From adea854d025d9acc9201268954bd43a8e71db964 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 12 Nov 2025 15:36:52 -0700 Subject: [PATCH 01/30] create new pr From 0b1944f2c604e75a25a8e33bac111a19f9fd1daa Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Thu, 13 Nov 2025 08:08:27 -0700 Subject: [PATCH 02/30] Use ipv4 instead of ipv6 in tests --- .../block_aggregator_api/src/api/protobuf_adapter/tests.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/crates/services/block_aggregator_api/src/api/protobuf_adapter/tests.rs b/crates/services/block_aggregator_api/src/api/protobuf_adapter/tests.rs index 380b74ef318..95ca3045562 100644 --- a/crates/services/block_aggregator_api/src/api/protobuf_adapter/tests.rs +++ b/crates/services/block_aggregator_api/src/api/protobuf_adapter/tests.rs @@ -34,12 +34,11 @@ use futures::{ StreamExt, TryStreamExt, }; -use std::net::TcpListener; fn free_local_addr() -> String { - let listener = TcpListener::bind("[::1]:0").unwrap(); - let addr = listener.local_addr().unwrap(); // OS picks a free port - format!("[::1]:{}", addr.port()) + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + format!("127.0.0.1:{}", addr.port()) } #[tokio::test] From 7f4533b9ef63166485252f982d9531d324a73733 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Thu, 13 Nov 2025 08:14:09 -0700 Subject: [PATCH 03/30] Make api optional --- crates/fuel-core/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/fuel-core/Cargo.toml b/crates/fuel-core/Cargo.toml index d33caef2b64..237cf528451 100644 --- a/crates/fuel-core/Cargo.toml +++ b/crates/fuel-core/Cargo.toml @@ -60,7 +60,7 @@ clap = { workspace = true, features = ["derive"] } cosmrs = { version = "0.21", optional = true } derive_more = { version = "0.99" } enum-iterator = { workspace = true } -fuel-core-block-aggregator-api = { workspace = true } +fuel-core-block-aggregator-api = { workspace = true, optional = true } fuel-core-chain-config = { workspace = true, features = ["std"] } fuel-core-compression-service = { workspace = true } fuel-core-consensus-module = { workspace = true } From b016b3e3bd438ccfd6bfd0c590afccf03047e889 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Thu, 13 Nov 2025 10:58:48 -0700 Subject: [PATCH 04/30] Add missing features for new db types including unique rollback impl --- bin/fuel-core/src/cli/rollback.rs | 2 +- crates/fuel-core/src/combined_database.rs | 83 ++++++++++++++++++++++- crates/fuel-core/src/database.rs | 29 ++++++++ 3 files changed, 111 insertions(+), 3 deletions(-) diff --git a/bin/fuel-core/src/cli/rollback.rs b/bin/fuel-core/src/cli/rollback.rs index cdc99f092fd..afeaf945718 100644 --- a/bin/fuel-core/src/cli/rollback.rs +++ b/bin/fuel-core/src/cli/rollback.rs @@ -59,7 +59,7 @@ pub async fn exec(command: Command) -> anyhow::Result<()> { use crate::cli::ShutdownListener; let path = command.database_path.as_path(); - let db = CombinedDatabase::open( + let mut db = CombinedDatabase::open( path, StateRewindPolicy::RewindFullRange, DatabaseConfig { diff --git a/crates/fuel-core/src/combined_database.rs b/crates/fuel-core/src/combined_database.rs index ac8ce32f48f..3e1472cfa2c 100644 --- a/crates/fuel-core/src/combined_database.rs +++ b/crates/fuel-core/src/combined_database.rs @@ -3,6 +3,7 @@ use crate::state::{ historical_rocksdb::StateRewindPolicy, rocks_db::DatabaseConfig, }; +use anyhow::anyhow; use crate::{ database::{ @@ -23,6 +24,7 @@ use crate::{ }, service::DbType, }; +use fuel_core_block_aggregator_api::db::table::LatestBlock; #[cfg(feature = "test-helpers")] use fuel_core_chain_config::{ StateConfig, @@ -30,7 +32,6 @@ use fuel_core_chain_config::{ }; #[cfg(feature = "backup")] use fuel_core_services::TraceErr; -use fuel_core_storage::Result as StorageResult; #[cfg(feature = "test-helpers")] use fuel_core_storage::tables::{ Coins, @@ -40,6 +41,11 @@ use fuel_core_storage::tables::{ ContractsState, Messages, }; +use fuel_core_storage::{ + Error as StorageError, + Result as StorageResult, + StorageAsRef, +}; use fuel_core_types::{ blockchain::primitives::DaBlockHeight, fuel_types::BlockHeight, @@ -96,6 +102,8 @@ impl CombinedDatabase { crate::state::rocks_db::RocksDb::::prune(path)?; crate::state::rocks_db::RocksDb::::prune(path)?; crate::state::rocks_db::RocksDb::::prune(path)?; + crate::state::rocks_db::RocksDb::::prune(path)?; + crate::state::rocks_db::RocksDb::::prune(path)?; Ok(()) } @@ -139,6 +147,16 @@ impl CombinedDatabase { crate::state::rocks_db::RocksDb::::backup(db_dir, temp_dir) .trace_err("Failed to backup compression database")?; + crate::state::rocks_db::RocksDb::::backup( + db_dir, temp_dir, + ) + .trace_err("Failed to backup block aggregation storage database")?; + + crate::state::rocks_db::RocksDb::::backup( + db_dir, temp_dir, + ) + .trace_err("Failed to backup block aggregation s3 database")?; + Ok(()) } @@ -193,6 +211,18 @@ impl CombinedDatabase { ) .trace_err("Failed to restore compression database")?; + crate::state::rocks_db::RocksDb::::restore( + temp_restore_dir, + backup_dir, + ) + .trace_err("Failed to restore block aggregation storage database")?; + + crate::state::rocks_db::RocksDb::::restore( + temp_restore_dir, + backup_dir, + ) + .trace_err("Failed to restore block aggregation s3 database")?; + Ok(()) } @@ -348,6 +378,8 @@ impl CombinedDatabase { self.relayer.check_version()?; self.gas_price.check_version()?; self.compression.check_version()?; + self.block_aggregation_storage.check_version()?; + self.block_aggregation_s3.check_version()?; Ok(()) } @@ -363,9 +395,20 @@ impl CombinedDatabase { &self.block_aggregation_storage } + pub fn block_aggregation_storage_mut( + &mut self, + ) -> &mut Database { + &mut self.block_aggregation_storage + } + pub fn block_aggregation_s3(&self) -> &Database { &self.block_aggregation_s3 } + pub fn block_aggregation_s3_mut( + &mut self, + ) -> &mut Database { + &mut self.block_aggregation_s3 + } #[cfg(any(feature = "test-helpers", test))] pub fn on_chain_mut(&mut self) -> &mut Database { @@ -445,7 +488,7 @@ impl CombinedDatabase { /// Rollbacks the state of the blockchain to a specific block height. pub fn rollback_to( - &self, + &mut self, target_block_height: BlockHeight, shutdown_listener: &mut S, ) -> anyhow::Result<()> @@ -473,6 +516,28 @@ impl CombinedDatabase { let compression_db_rolled_back = is_equal_or_none(compression_db_height, target_block_height); + let block_aggregation_storage_height = self + .block_aggregation_storage() + .storage_as_ref::() + .get(&()) + .map_err(|e: StorageError| anyhow!(e))? + .map(|b| b.into_owned()); + let block_aggregation_storage_rolled_back = is_equal_or_less_than_or_none( + block_aggregation_storage_height, + target_block_height, + ); + + let block_aggregation_s3_height = self + .block_aggregation_s3() + .storage_as_ref::() + .get(&()) + .map_err(|e: StorageError| anyhow!(e))? + .map(|b| b.into_owned()); + let block_aggregation_s3_rolled_back = is_equal_or_less_than_or_none( + block_aggregation_s3_height, + target_block_height, + ); + if on_chain_height == target_block_height && off_chain_height == target_block_height && gas_price_rolled_back @@ -532,6 +597,16 @@ impl CombinedDatabase { { self.compression().rollback_last_block()?; } + + if !block_aggregation_storage_rolled_back { + self.block_aggregation_storage_mut() + .rollback_to(target_block_height)?; + } + + if !block_aggregation_s3_rolled_back { + self.block_aggregation_s3_mut() + .rollback_to(target_block_height)?; + } } if shutdown_listener.is_cancelled() { @@ -669,6 +744,10 @@ fn is_equal_or_none(maybe_left: Option, right: T) -> bool { maybe_left.map(|left| left == right).unwrap_or(true) } +fn is_equal_or_less_than_or_none(maybe_left: Option, right: T) -> bool { + maybe_left.map(|left| left <= right).unwrap_or(true) +} + #[allow(non_snake_case)] #[cfg(feature = "backup")] #[cfg(test)] diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index 96b03caad7b..bc712edf169 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -93,6 +93,9 @@ use crate::{ }, state::HeightType, }; +use anyhow::anyhow; +use fuel_core_block_aggregator_api::db::table::LatestBlock; +use fuel_core_storage::transactional::WriteTransaction; #[cfg(feature = "rocksdb")] use std::path::Path; @@ -447,16 +450,42 @@ impl Modifiable for Database { impl Modifiable for Database { fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { + // Does not need to be monotonically increasing because + // storage values are modified in parallel from different heights commit_changes_with_height_update(self, changes, |_iter| Ok(Vec::new())) } } +impl Database { + pub fn rollback_to(&mut self, block_height: BlockHeight) -> StorageResult<()> { + let mut tx = self.write_transaction(); + tx.storage_as_mut::() + .insert(&(), &block_height) + .map_err(|e: StorageError| anyhow!(e))?; + tx.commit().map_err(|e: StorageError| anyhow!(e))?; + Ok(()) + } +} + impl Modifiable for Database { fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { + // Does not need to be monotonically increasing because + // storage values are modified in parallel from different heights commit_changes_with_height_update(self, changes, |_iter| Ok(Vec::new())) } } +impl Database { + pub fn rollback_to(&mut self, block_height: BlockHeight) -> StorageResult<()> { + let mut tx = self.write_transaction(); + tx.storage_as_mut::() + .insert(&(), &block_height) + .map_err(|e: StorageError| anyhow!(e))?; + tx.commit().map_err(|e: StorageError| anyhow!(e))?; + Ok(()) + } +} + #[cfg(feature = "relayer")] impl Modifiable for Database { fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { From a2161d202ed7dbeaaee3e7800fdbb57088453446 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Thu, 13 Nov 2025 11:07:29 -0700 Subject: [PATCH 05/30] Remove unused marker --- crates/types/src/test_helpers.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/types/src/test_helpers.rs b/crates/types/src/test_helpers.rs index 37e187f6b81..8c64ae90a10 100644 --- a/crates/types/src/test_helpers.rs +++ b/crates/types/src/test_helpers.rs @@ -72,7 +72,6 @@ pub fn create_contract( (tx, contract_id) } -#[allow(unused)] fn arb_txs() -> impl Strategy> { prop::collection::vec(arb_transaction(), 0..10) } From c032a1aa1bb26edab2636957b8b80c6d67df7743 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Thu, 13 Nov 2025 11:16:52 -0700 Subject: [PATCH 06/30] Improve policy conversion --- .../serializer_adapter.rs | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter.rs b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter.rs index b497145f2da..aadc179b081 100644 --- a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter.rs +++ b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter.rs @@ -609,25 +609,23 @@ fn proto_policies_from_policies( policies: &fuel_core_types::fuel_tx::policies::Policies, ) -> ProtoPolicies { let mut values = [0u64; 6]; - if policies.is_set(PolicyType::Tip) { - values[0] = policies.get(PolicyType::Tip).unwrap_or_default(); + if let Some(value) = policies.get(PolicyType::Tip) { + values[0] = value; } - if policies.is_set(PolicyType::WitnessLimit) { - let value = policies.get(PolicyType::WitnessLimit).unwrap_or_default(); + if let Some(value) = policies.get(PolicyType::WitnessLimit) { values[1] = value; } - if policies.is_set(PolicyType::Maturity) { - let value = policies.get(PolicyType::Maturity).unwrap_or_default(); + if let Some(value) = policies.get(PolicyType::Maturity) { values[2] = value; } - if policies.is_set(PolicyType::MaxFee) { - values[3] = policies.get(PolicyType::MaxFee).unwrap_or_default(); + if let Some(value) = policies.get(PolicyType::MaxFee) { + values[3] = value; } - if policies.is_set(PolicyType::Expiration) { - values[4] = policies.get(PolicyType::Expiration).unwrap_or_default(); + if let Some(value) = policies.get(PolicyType::Expiration) { + values[4] = value; } - if policies.is_set(PolicyType::Owner) { - values[5] = policies.get(PolicyType::Owner).unwrap_or_default(); + if let Some(value) = policies.get(PolicyType::Owner) { + values[5] = value; } let bits = policies.bits(); ProtoPolicies { From b2ba2a26ccb922433671bd3b2a0439d909027370 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Thu, 13 Nov 2025 11:46:24 -0700 Subject: [PATCH 07/30] Reduce size of policies vec if not used --- .../blocks/importer_and_db_source/serializer_adapter.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter.rs b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter.rs index aadc179b081..c0cf1e24634 100644 --- a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter.rs +++ b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter.rs @@ -609,25 +609,33 @@ fn proto_policies_from_policies( policies: &fuel_core_types::fuel_tx::policies::Policies, ) -> ProtoPolicies { let mut values = [0u64; 6]; + let mut truncated_len = 0; if let Some(value) = policies.get(PolicyType::Tip) { values[0] = value; + truncated_len = 1; } if let Some(value) = policies.get(PolicyType::WitnessLimit) { values[1] = value; + truncated_len = 2; } if let Some(value) = policies.get(PolicyType::Maturity) { values[2] = value; + truncated_len = 3; } if let Some(value) = policies.get(PolicyType::MaxFee) { values[3] = value; + truncated_len = 4; } if let Some(value) = policies.get(PolicyType::Expiration) { values[4] = value; + truncated_len = 5; } if let Some(value) = policies.get(PolicyType::Owner) { values[5] = value; + truncated_len = 6; } let bits = policies.bits(); + values[..truncated_len].to_vec(); ProtoPolicies { bits, values: values.to_vec(), From 107a46cfca32e91720c46ad120bca1bce6f37f34 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Thu, 13 Nov 2025 16:51:10 -0700 Subject: [PATCH 08/30] Use references in serializer adapter code --- .../serializer_adapter.rs | 93 +++++++------------ 1 file changed, 35 insertions(+), 58 deletions(-) diff --git a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter.rs b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter.rs index c0cf1e24634..f3db703fb06 100644 --- a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter.rs +++ b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter.rs @@ -125,13 +125,16 @@ impl BlockSerializer for SerializerAdapter { type Block = ProtoBlock; fn serialize_block(&self, block: &FuelBlock) -> crate::result::Result { - let (header, txs) = block.clone().into_inner(); - let proto_header = proto_header_from_header(header); + let proto_header = proto_header_from_header(block.header()); match &block { FuelBlock::V1(_) => { let proto_v1_block = ProtoV1Block { header: Some(proto_header), - transactions: txs.into_iter().map(proto_tx_from_tx).collect(), + transactions: block + .transactions() + .into_iter() + .map(proto_tx_from_tx) + .collect(), }; Ok(ProtoBlock { versioned_block: Some(ProtoVersionedBlock::V1(proto_v1_block)), @@ -141,19 +144,19 @@ impl BlockSerializer for SerializerAdapter { } } -fn proto_header_from_header(header: BlockHeader) -> ProtoHeader { +fn proto_header_from_header(header: &BlockHeader) -> ProtoHeader { let block_id = header.id(); - let consensus = *header.consensus(); + let consensus = header.consensus(); let versioned_header = match header { BlockHeader::V1(header) => { let proto_v1_header = - proto_v1_header_from_v1_header(consensus, block_id, header); + proto_v1_header_from_v1_header(&consensus, &block_id, header); ProtoVersionedHeader::V1(proto_v1_header) } #[cfg(feature = "fault-proving")] BlockHeader::V2(header) => { let proto_v2_header = - proto_v2_header_from_v2_header(consensus, block_id, header); + proto_v2_header_from_v2_header(consensus, &block_id, header); ProtoVersionedHeader::V2(proto_v2_header) } }; @@ -164,9 +167,9 @@ fn proto_header_from_header(header: BlockHeader) -> ProtoHeader { } fn proto_v1_header_from_v1_header( - consensus: ConsensusHeader, - block_id: BlockId, - header: BlockHeaderV1, + consensus: &ConsensusHeader, + block_id: &BlockId, + header: &BlockHeaderV1, ) -> ProtoV1Header { let application = header.application(); let generated = application.generated; @@ -190,9 +193,9 @@ fn proto_v1_header_from_v1_header( #[cfg(feature = "fault-proving")] fn proto_v2_header_from_v2_header( - consensus: ConsensusHeader, - block_id: BlockId, - header: BlockHeaderV2, + consensus: &ConsensusHeader, + block_id: &BlockId, + header: &BlockHeaderV2, ) -> ProtoV2Header { let application = *header.application(); let generated = application.generated; @@ -215,7 +218,7 @@ fn proto_v2_header_from_v2_header( } } -fn proto_tx_from_tx(tx: FuelTransaction) -> ProtoTransaction { +fn proto_tx_from_tx(tx: &FuelTransaction) -> ProtoTransaction { match tx { FuelTransaction::Script(script) => { let proto_script = ProtoScriptTx { @@ -224,16 +227,10 @@ fn proto_tx_from_tx(tx: FuelTransaction) -> ProtoTransaction { script: script.script().clone(), script_data: script.script_data().clone(), policies: Some(proto_policies_from_policies(script.policies())), - inputs: script - .inputs() - .iter() - .cloned() - .map(proto_input_from_input) - .collect(), + inputs: script.inputs().iter().map(proto_input_from_input).collect(), outputs: script .outputs() .iter() - .cloned() .map(proto_output_from_output) .collect(), witnesses: script @@ -258,16 +255,10 @@ fn proto_tx_from_tx(tx: FuelTransaction) -> ProtoTransaction { .map(proto_storage_slot_from_storage_slot) .collect(), policies: Some(proto_policies_from_policies(create.policies())), - inputs: create - .inputs() - .iter() - .cloned() - .map(proto_input_from_input) - .collect(), + inputs: create.inputs().iter().map(proto_input_from_input).collect(), outputs: create .outputs() .iter() - .cloned() .map(proto_output_from_output) .collect(), witnesses: create @@ -308,13 +299,11 @@ fn proto_tx_from_tx(tx: FuelTransaction) -> ProtoTransaction { inputs: upgrade .inputs() .iter() - .cloned() .map(proto_input_from_input) .collect(), outputs: upgrade .outputs() .iter() - .cloned() .map(proto_output_from_output) .collect(), witnesses: upgrade @@ -337,16 +326,10 @@ fn proto_tx_from_tx(tx: FuelTransaction) -> ProtoTransaction { subsections_number: u32::from(*upload.subsections_number()), proof_set: upload.proof_set().iter().map(bytes32_to_vec).collect(), policies: Some(proto_policies_from_policies(upload.policies())), - inputs: upload - .inputs() - .iter() - .cloned() - .map(proto_input_from_input) - .collect(), + inputs: upload.inputs().iter().map(proto_input_from_input).collect(), outputs: upload .outputs() .iter() - .cloned() .map(proto_output_from_output) .collect(), witnesses: upload @@ -366,16 +349,10 @@ fn proto_tx_from_tx(tx: FuelTransaction) -> ProtoTransaction { blob_id: blob.blob_id().as_ref().to_vec(), witness_index: u32::from(*blob.bytecode_witness_index()), policies: Some(proto_policies_from_policies(blob.policies())), - inputs: blob - .inputs() - .iter() - .cloned() - .map(proto_input_from_input) - .collect(), + inputs: blob.inputs().iter().map(proto_input_from_input).collect(), outputs: blob .outputs() .iter() - .cloned() .map(proto_output_from_output) .collect(), witnesses: blob @@ -393,7 +370,7 @@ fn proto_tx_from_tx(tx: FuelTransaction) -> ProtoTransaction { } } -fn proto_input_from_input(input: Input) -> ProtoInput { +fn proto_input_from_input(input: &Input) -> ProtoInput { match input { Input::CoinSigned(coin_signed) => ProtoInput { variant: Some(ProtoInputVariant::CoinSigned(ProtoCoinSignedInput { @@ -536,7 +513,7 @@ fn proto_contract_output_from_contract( } } -fn proto_output_from_output(output: Output) -> ProtoOutput { +fn proto_output_from_output(output: &Output) -> ProtoOutput { let variant = match output { Output::Coin { to, @@ -544,7 +521,7 @@ fn proto_output_from_output(output: Output) -> ProtoOutput { asset_id, } => ProtoOutputVariant::Coin(ProtoCoinOutput { to: to.as_ref().to_vec(), - amount, + amount: *amount, asset_id: asset_id.as_ref().to_vec(), }), Output::Contract(contract) => { @@ -556,7 +533,7 @@ fn proto_output_from_output(output: Output) -> ProtoOutput { asset_id, } => ProtoOutputVariant::Change(ProtoChangeOutput { to: to.as_ref().to_vec(), - amount, + amount: *amount, asset_id: asset_id.as_ref().to_vec(), }), Output::Variable { @@ -565,7 +542,7 @@ fn proto_output_from_output(output: Output) -> ProtoOutput { asset_id, } => ProtoOutputVariant::Variable(ProtoVariableOutput { to: to.as_ref().to_vec(), - amount, + amount: *amount, asset_id: asset_id.as_ref().to_vec(), }), Output::ContractCreated { @@ -846,7 +823,7 @@ pub fn fuel_block_from_protobuf( .clone() .ok_or_else(|| anyhow::anyhow!("Missing protobuf header")) .map_err(Error::Serialization)?; - partial_header_from_proto_header(proto_header)? + partial_header_from_proto_header(&proto_header)? } }; let txs = match versioned_block { @@ -869,7 +846,7 @@ pub fn fuel_block_from_protobuf( } pub fn partial_header_from_proto_header( - proto_header: ProtoHeader, + proto_header: &ProtoHeader, ) -> Result { let partial_header = PartialBlockHeader { consensus: proto_header_to_empty_consensus_header(&proto_header)?, @@ -889,7 +866,7 @@ pub fn tx_from_proto_tx(proto_tx: &ProtoTransaction) -> Result let policies = proto_script .policies .clone() - .map(policies_from_proto_policies) + .map(|p| policies_from_proto_policies(&p)) .unwrap_or_default(); let inputs = proto_script .inputs @@ -931,7 +908,7 @@ pub fn tx_from_proto_tx(proto_tx: &ProtoTransaction) -> Result let policies = proto_create .policies .clone() - .map(policies_from_proto_policies) + .map(|p| policies_from_proto_policies(&p)) .unwrap_or_default(); let inputs = proto_create .inputs @@ -1019,7 +996,7 @@ pub fn tx_from_proto_tx(proto_tx: &ProtoTransaction) -> Result let policies = proto_upgrade .policies .clone() - .map(policies_from_proto_policies) + .map(|p| policies_from_proto_policies(&p)) .unwrap_or_default(); let inputs = proto_upgrade .inputs @@ -1051,7 +1028,7 @@ pub fn tx_from_proto_tx(proto_tx: &ProtoTransaction) -> Result let policies = proto_upload .policies .clone() - .map(policies_from_proto_policies) + .map(|p| policies_from_proto_policies(&p)) .unwrap_or_default(); let inputs = proto_upload .inputs @@ -1125,7 +1102,7 @@ pub fn tx_from_proto_tx(proto_tx: &ProtoTransaction) -> Result let policies = proto_blob .policies .clone() - .map(policies_from_proto_policies) + .map(|p| policies_from_proto_policies(&p)) .unwrap_or_default(); let inputs = proto_blob .inputs @@ -1384,11 +1361,11 @@ fn input_from_proto_input(proto_input: &ProtoInput) -> Result { } } -fn policies_from_proto_policies(proto_policies: ProtoPolicies) -> FuelPolicies { +fn policies_from_proto_policies(proto_policies: &ProtoPolicies) -> FuelPolicies { let ProtoPolicies { bits, values } = proto_policies; let mut policies = FuelPolicies::default(); let bits = - PoliciesBits::from_bits(bits).expect("Should be able to create from `u32`"); + PoliciesBits::from_bits(*bits).expect("Should be able to create from `u32`"); if bits.contains(PoliciesBits::Tip) && let Some(tip) = values.first() { From ae193fef165ef382596f8b488121ac0e286f6cdc Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Fri, 14 Nov 2025 08:36:42 -0700 Subject: [PATCH 09/30] add fuel-core-protobuf after reverting old changes --- Cargo.lock | 13 +++++++++++++ Cargo.toml | 1 + crates/services/block_aggregator_api/Cargo.toml | 1 + .../serializer_adapter/proto_to_fuel_conversions.rs | 0 .../block_aggregator_api/src/protobuf_types.rs | 2 +- 5 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/proto_to_fuel_conversions.rs diff --git a/Cargo.lock b/Cargo.lock index 2c8fae248ed..fc7e2004e83 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4249,6 +4249,7 @@ dependencies = [ "aws-smithy-mocks", "bytes", "enum-iterator", + "fuel-core-protobuf", "fuel-core-services", "fuel-core-storage", "fuel-core-types 0.47.1", @@ -4616,6 +4617,18 @@ dependencies = [ "tracing", ] +[[package]] +name = "fuel-core-protobuf" +version = "0.1.0" +source = "git+https://github.com/FuelLabs/fuel-core-protobuf.git?branch=add-defs#5b6e4d6c6db0ac654f152423cb0a6292fac9c7dc" +dependencies = [ + "prost 0.14.1", + "serde", + "tonic 0.14.2", + "tonic-prost", + "tonic-prost-build", +] + [[package]] name = "fuel-core-provider" version = "0.47.1" diff --git a/Cargo.toml b/Cargo.toml index 96bd7d0ef0c..1142c760289 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -110,6 +110,7 @@ fuel-core-p2p = { version = "0.47.1", path = "./crates/services/p2p" } fuel-core-parallel-executor = { version = "0.47.1", path = "./crates/services/parallel-executor" } fuel-core-poa = { version = "0.47.1", path = "./crates/services/consensus_module/poa" } fuel-core-producer = { version = "0.47.1", path = "./crates/services/producer" } +fuel-core-protobuf = { version = "0.1.0", git = "https://github.com/FuelLabs/fuel-core-protobuf.git", branch = "add-defs" } fuel-core-provider = { version = "0.47.1", path = "./crates/provider" } fuel-core-relayer = { version = "0.47.1", path = "./crates/services/relayer" } fuel-core-services = { version = "0.47.1", path = "./crates/services" } diff --git a/crates/services/block_aggregator_api/Cargo.toml b/crates/services/block_aggregator_api/Cargo.toml index a9c8269046d..aa0677686bf 100644 --- a/crates/services/block_aggregator_api/Cargo.toml +++ b/crates/services/block_aggregator_api/Cargo.toml @@ -20,6 +20,7 @@ aws-sdk-s3 = "1.111.0" aws-smithy-mocks = "0.2.0" bytes = { workspace = true, features = ["serde"] } enum-iterator = { workspace = true } +fuel-core-protobuf = { workspace = true } fuel-core-services = { workspace = true } fuel-core-storage = { workspace = true, features = ["std"] } fuel-core-types = { workspace = true, features = ["std"] } diff --git a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/proto_to_fuel_conversions.rs b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/proto_to_fuel_conversions.rs new file mode 100644 index 00000000000..e69de29bb2d diff --git a/crates/services/block_aggregator_api/src/protobuf_types.rs b/crates/services/block_aggregator_api/src/protobuf_types.rs index 648ac0e278d..256ed21a634 100644 --- a/crates/services/block_aggregator_api/src/protobuf_types.rs +++ b/crates/services/block_aggregator_api/src/protobuf_types.rs @@ -1 +1 @@ -tonic::include_proto!("blockaggregator"); +pub use fuel_core_protobuf::*; From ffd326a913cfad7b93872bdf5ff0fe36d3bd2a05 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Fri, 14 Nov 2025 08:54:50 -0700 Subject: [PATCH 10/30] Move functions to their own files --- .../serializer_adapter.rs | 1428 +---------------- .../fuel_to_proto_conversions.rs | 564 +++++++ .../proto_to_fuel_conversions.rs | 919 +++++++++++ 3 files changed, 1487 insertions(+), 1424 deletions(-) create mode 100644 crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/fuel_to_proto_conversions.rs diff --git a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter.rs b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter.rs index f3db703fb06..57a08b0b30a 100644 --- a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter.rs +++ b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter.rs @@ -3,51 +3,11 @@ use crate::protobuf_types::V2Header as ProtoV2Header; use crate::{ blocks::importer_and_db_source::BlockSerializer, protobuf_types::{ - BlobTransaction as ProtoBlobTx, Block as ProtoBlock, - ChangeOutput as ProtoChangeOutput, - CoinOutput as ProtoCoinOutput, - CoinPredicateInput as ProtoCoinPredicateInput, - CoinSignedInput as ProtoCoinSignedInput, - ContractCreatedOutput as ProtoContractCreatedOutput, - ContractInput as ProtoContractInput, - ContractOutput as ProtoContractOutput, - CreateTransaction as ProtoCreateTx, - Header as ProtoHeader, - Input as ProtoInput, - MessageCoinPredicateInput as ProtoMessageCoinPredicateInput, - MessageCoinSignedInput as ProtoMessageCoinSignedInput, - MessageDataPredicateInput as ProtoMessageDataPredicateInput, - MessageDataSignedInput as ProtoMessageDataSignedInput, - MintTransaction as ProtoMintTx, - Output as ProtoOutput, - Policies as ProtoPolicies, - ScriptTransaction as ProtoScriptTx, - StorageSlot as ProtoStorageSlot, - Transaction as ProtoTransaction, - TxPointer as ProtoTxPointer, - UpgradeConsensusParameters as ProtoUpgradeConsensusParameters, - UpgradePurpose as ProtoUpgradePurpose, - UpgradeStateTransition as ProtoUpgradeStateTransition, - UpgradeTransaction as ProtoUpgradeTx, - UploadTransaction as ProtoUploadTx, - UtxoId as ProtoUtxoId, V1Block as ProtoV1Block, - V1Header as ProtoV1Header, - VariableOutput as ProtoVariableOutput, block::VersionedBlock as ProtoVersionedBlock, - header::VersionedHeader as ProtoVersionedHeader, - input::Variant as ProtoInputVariant, - output::Variant as ProtoOutputVariant, - transaction::Variant as ProtoTransactionVariant, - upgrade_purpose::Variant as ProtoUpgradePurposeVariant, - }, - result::{ - Error, - Result, }, }; -use anyhow::anyhow; #[cfg(feature = "fault-proving")] use fuel_core_types::{ blockchain::header::BlockHeaderV2, @@ -57,66 +17,9 @@ use fuel_core_types::{ use fuel_core_types::{ blockchain::{ block::Block as FuelBlock, - header::{ - ApplicationHeader, - BlockHeader, - BlockHeaderV1, - ConsensusHeader, - GeneratedConsensusFields, - PartialBlockHeader, - }, - primitives::{ - BlockId, - DaBlockHeight, - Empty, - }, - }, - fuel_tx::{ - Address, - BlobBody, - Bytes32, - Input, - Output, - StorageSlot, - Transaction as FuelTransaction, - TxPointer, - UpgradePurpose, - UploadBody, - UtxoId, - Witness, - field::{ - BlobId as _, - BytecodeRoot as _, - BytecodeWitnessIndex as _, - InputContract as _, - Inputs, - MintAmount as _, - MintAssetId as _, - MintGasPrice as _, - OutputContract as _, - Outputs, - Policies as _, - ProofSet as _, - ReceiptsRoot as _, - Salt as _, - Script as _, - ScriptData as _, - ScriptGasLimit as _, - StorageSlots as _, - SubsectionIndex as _, - SubsectionsNumber as _, - TxPointer as TxPointerField, - UpgradePurpose as UpgradePurposeField, - Witnesses as _, - }, - policies::{ - Policies as FuelPolicies, - PoliciesBits, - PolicyType, - }, }, - tai64, }; +use crate::blocks::importer_and_db_source::serializer_adapter::fuel_to_proto_conversions::{proto_header_from_header, proto_tx_from_tx}; #[derive(Clone)] pub struct SerializerAdapter; @@ -144,1332 +47,8 @@ impl BlockSerializer for SerializerAdapter { } } -fn proto_header_from_header(header: &BlockHeader) -> ProtoHeader { - let block_id = header.id(); - let consensus = header.consensus(); - let versioned_header = match header { - BlockHeader::V1(header) => { - let proto_v1_header = - proto_v1_header_from_v1_header(&consensus, &block_id, header); - ProtoVersionedHeader::V1(proto_v1_header) - } - #[cfg(feature = "fault-proving")] - BlockHeader::V2(header) => { - let proto_v2_header = - proto_v2_header_from_v2_header(consensus, &block_id, header); - ProtoVersionedHeader::V2(proto_v2_header) - } - }; - - ProtoHeader { - versioned_header: Some(versioned_header), - } -} - -fn proto_v1_header_from_v1_header( - consensus: &ConsensusHeader, - block_id: &BlockId, - header: &BlockHeaderV1, -) -> ProtoV1Header { - let application = header.application(); - let generated = application.generated; - - ProtoV1Header { - da_height: application.da_height.0, - consensus_parameters_version: application.consensus_parameters_version, - state_transition_bytecode_version: application.state_transition_bytecode_version, - transactions_count: u32::from(generated.transactions_count), - message_receipt_count: generated.message_receipt_count, - transactions_root: bytes32_to_vec(&generated.transactions_root), - message_outbox_root: bytes32_to_vec(&generated.message_outbox_root), - event_inbox_root: bytes32_to_vec(&generated.event_inbox_root), - prev_root: bytes32_to_vec(&consensus.prev_root), - height: u32::from(consensus.height), - time: consensus.time.0, - application_hash: bytes32_to_vec(&consensus.generated.application_hash), - block_id: Some(block_id.as_slice().to_vec()), - } -} - -#[cfg(feature = "fault-proving")] -fn proto_v2_header_from_v2_header( - consensus: &ConsensusHeader, - block_id: &BlockId, - header: &BlockHeaderV2, -) -> ProtoV2Header { - let application = *header.application(); - let generated = application.generated; - - ProtoV2Header { - da_height: application.da_height.0, - consensus_parameters_version: application.consensus_parameters_version, - state_transition_bytecode_version: application.state_transition_bytecode_version, - transactions_count: u32::from(generated.transactions_count), - message_receipt_count: generated.message_receipt_count, - transactions_root: bytes32_to_vec(&generated.transactions_root), - message_outbox_root: bytes32_to_vec(&generated.message_outbox_root), - event_inbox_root: bytes32_to_vec(&generated.event_inbox_root), - tx_id_commitment: bytes32_to_vec(&generated.tx_id_commitment), - prev_root: bytes32_to_vec(&consensus.prev_root), - height: u32::from(consensus.height), - time: consensus.time.0, - application_hash: bytes32_to_vec(&consensus.generated.application_hash), - block_id: Some(block_id.as_slice().to_vec()), - } -} - -fn proto_tx_from_tx(tx: &FuelTransaction) -> ProtoTransaction { - match tx { - FuelTransaction::Script(script) => { - let proto_script = ProtoScriptTx { - script_gas_limit: *script.script_gas_limit(), - receipts_root: bytes32_to_vec(script.receipts_root()), - script: script.script().clone(), - script_data: script.script_data().clone(), - policies: Some(proto_policies_from_policies(script.policies())), - inputs: script.inputs().iter().map(proto_input_from_input).collect(), - outputs: script - .outputs() - .iter() - .map(proto_output_from_output) - .collect(), - witnesses: script - .witnesses() - .iter() - .map(|witness| witness.as_ref().to_vec()) - .collect(), - metadata: None, - }; - - ProtoTransaction { - variant: Some(ProtoTransactionVariant::Script(proto_script)), - } - } - FuelTransaction::Create(create) => { - let proto_create = ProtoCreateTx { - bytecode_witness_index: u32::from(*create.bytecode_witness_index()), - salt: create.salt().as_ref().to_vec(), - storage_slots: create - .storage_slots() - .iter() - .map(proto_storage_slot_from_storage_slot) - .collect(), - policies: Some(proto_policies_from_policies(create.policies())), - inputs: create.inputs().iter().map(proto_input_from_input).collect(), - outputs: create - .outputs() - .iter() - .map(proto_output_from_output) - .collect(), - witnesses: create - .witnesses() - .iter() - .map(|witness| witness.as_ref().to_vec()) - .collect(), - metadata: None, - }; - - ProtoTransaction { - variant: Some(ProtoTransactionVariant::Create(proto_create)), - } - } - FuelTransaction::Mint(mint) => { - let proto_mint = ProtoMintTx { - tx_pointer: Some(proto_tx_pointer(mint.tx_pointer())), - input_contract: Some(proto_contract_input_from_contract( - mint.input_contract(), - )), - output_contract: Some(proto_contract_output_from_contract( - mint.output_contract(), - )), - mint_amount: *mint.mint_amount(), - mint_asset_id: mint.mint_asset_id().as_ref().to_vec(), - gas_price: *mint.gas_price(), - metadata: None, - }; - - ProtoTransaction { - variant: Some(ProtoTransactionVariant::Mint(proto_mint)), - } - } - FuelTransaction::Upgrade(upgrade) => { - let proto_upgrade = ProtoUpgradeTx { - purpose: Some(proto_upgrade_purpose(upgrade.upgrade_purpose())), - policies: Some(proto_policies_from_policies(upgrade.policies())), - inputs: upgrade - .inputs() - .iter() - .map(proto_input_from_input) - .collect(), - outputs: upgrade - .outputs() - .iter() - .map(proto_output_from_output) - .collect(), - witnesses: upgrade - .witnesses() - .iter() - .map(|witness| witness.as_ref().to_vec()) - .collect(), - metadata: None, - }; - - ProtoTransaction { - variant: Some(ProtoTransactionVariant::Upgrade(proto_upgrade)), - } - } - FuelTransaction::Upload(upload) => { - let proto_upload = ProtoUploadTx { - root: bytes32_to_vec(upload.bytecode_root()), - witness_index: u32::from(*upload.bytecode_witness_index()), - subsection_index: u32::from(*upload.subsection_index()), - subsections_number: u32::from(*upload.subsections_number()), - proof_set: upload.proof_set().iter().map(bytes32_to_vec).collect(), - policies: Some(proto_policies_from_policies(upload.policies())), - inputs: upload.inputs().iter().map(proto_input_from_input).collect(), - outputs: upload - .outputs() - .iter() - .map(proto_output_from_output) - .collect(), - witnesses: upload - .witnesses() - .iter() - .map(|witness| witness.as_ref().to_vec()) - .collect(), - metadata: None, - }; - - ProtoTransaction { - variant: Some(ProtoTransactionVariant::Upload(proto_upload)), - } - } - FuelTransaction::Blob(blob) => { - let proto_blob = ProtoBlobTx { - blob_id: blob.blob_id().as_ref().to_vec(), - witness_index: u32::from(*blob.bytecode_witness_index()), - policies: Some(proto_policies_from_policies(blob.policies())), - inputs: blob.inputs().iter().map(proto_input_from_input).collect(), - outputs: blob - .outputs() - .iter() - .map(proto_output_from_output) - .collect(), - witnesses: blob - .witnesses() - .iter() - .map(|witness| witness.as_ref().to_vec()) - .collect(), - metadata: None, - }; - - ProtoTransaction { - variant: Some(ProtoTransactionVariant::Blob(proto_blob)), - } - } - } -} - -fn proto_input_from_input(input: &Input) -> ProtoInput { - match input { - Input::CoinSigned(coin_signed) => ProtoInput { - variant: Some(ProtoInputVariant::CoinSigned(ProtoCoinSignedInput { - utxo_id: Some(proto_utxo_id_from_utxo_id(&coin_signed.utxo_id)), - owner: coin_signed.owner.as_ref().to_vec(), - amount: coin_signed.amount, - asset_id: coin_signed.asset_id.as_ref().to_vec(), - tx_pointer: Some(proto_tx_pointer(&coin_signed.tx_pointer)), - witness_index: coin_signed.witness_index.into(), - predicate_gas_used: 0, - predicate: vec![], - predicate_data: vec![], - })), - }, - Input::CoinPredicate(coin_predicate) => ProtoInput { - variant: Some(ProtoInputVariant::CoinPredicate(ProtoCoinPredicateInput { - utxo_id: Some(proto_utxo_id_from_utxo_id(&coin_predicate.utxo_id)), - owner: coin_predicate.owner.as_ref().to_vec(), - amount: coin_predicate.amount, - asset_id: coin_predicate.asset_id.as_ref().to_vec(), - tx_pointer: Some(proto_tx_pointer(&coin_predicate.tx_pointer)), - witness_index: 0, - predicate_gas_used: coin_predicate.predicate_gas_used, - predicate: coin_predicate.predicate.as_ref().to_vec(), - predicate_data: coin_predicate.predicate_data.as_ref().to_vec(), - })), - }, - Input::Contract(contract) => ProtoInput { - variant: Some(ProtoInputVariant::Contract(ProtoContractInput { - utxo_id: Some(proto_utxo_id_from_utxo_id(&contract.utxo_id)), - balance_root: bytes32_to_vec(&contract.balance_root), - state_root: bytes32_to_vec(&contract.state_root), - tx_pointer: Some(proto_tx_pointer(&contract.tx_pointer)), - contract_id: contract.contract_id.as_ref().to_vec(), - })), - }, - Input::MessageCoinSigned(message) => ProtoInput { - variant: Some(ProtoInputVariant::MessageCoinSigned( - ProtoMessageCoinSignedInput { - sender: message.sender.as_ref().to_vec(), - recipient: message.recipient.as_ref().to_vec(), - amount: message.amount, - nonce: message.nonce.as_ref().to_vec(), - witness_index: message.witness_index.into(), - predicate_gas_used: 0, - data: Vec::new(), - predicate: Vec::new(), - predicate_data: Vec::new(), - }, - )), - }, - Input::MessageCoinPredicate(message) => ProtoInput { - variant: Some(ProtoInputVariant::MessageCoinPredicate( - ProtoMessageCoinPredicateInput { - sender: message.sender.as_ref().to_vec(), - recipient: message.recipient.as_ref().to_vec(), - amount: message.amount, - nonce: message.nonce.as_ref().to_vec(), - witness_index: 0, - predicate_gas_used: message.predicate_gas_used, - data: Vec::new(), - predicate: message.predicate.as_ref().to_vec(), - predicate_data: message.predicate_data.as_ref().to_vec(), - }, - )), - }, - Input::MessageDataSigned(message) => ProtoInput { - variant: Some(ProtoInputVariant::MessageDataSigned( - ProtoMessageDataSignedInput { - sender: message.sender.as_ref().to_vec(), - recipient: message.recipient.as_ref().to_vec(), - amount: message.amount, - nonce: message.nonce.as_ref().to_vec(), - witness_index: message.witness_index.into(), - predicate_gas_used: 0, - data: message.data.as_ref().to_vec(), - predicate: Vec::new(), - predicate_data: Vec::new(), - }, - )), - }, - Input::MessageDataPredicate(message) => ProtoInput { - variant: Some(ProtoInputVariant::MessageDataPredicate( - ProtoMessageDataPredicateInput { - sender: message.sender.as_ref().to_vec(), - recipient: message.recipient.as_ref().to_vec(), - amount: message.amount, - nonce: message.nonce.as_ref().to_vec(), - witness_index: 0, - predicate_gas_used: message.predicate_gas_used, - data: message.data.as_ref().to_vec(), - predicate: message.predicate.as_ref().to_vec(), - predicate_data: message.predicate_data.as_ref().to_vec(), - }, - )), - }, - } -} - -fn proto_utxo_id_from_utxo_id(utxo_id: &UtxoId) -> ProtoUtxoId { - ProtoUtxoId { - tx_id: utxo_id.tx_id().as_ref().to_vec(), - output_index: utxo_id.output_index().into(), - } -} - -fn proto_tx_pointer(tx_pointer: &TxPointer) -> ProtoTxPointer { - ProtoTxPointer { - block_height: tx_pointer.block_height().into(), - tx_index: tx_pointer.tx_index().into(), - } -} - -fn proto_storage_slot_from_storage_slot(slot: &StorageSlot) -> ProtoStorageSlot { - ProtoStorageSlot { - key: slot.key().as_ref().to_vec(), - value: slot.value().as_ref().to_vec(), - } -} - -fn proto_contract_input_from_contract( - contract: &fuel_core_types::fuel_tx::input::contract::Contract, -) -> ProtoContractInput { - ProtoContractInput { - utxo_id: Some(proto_utxo_id_from_utxo_id(&contract.utxo_id)), - balance_root: bytes32_to_vec(&contract.balance_root), - state_root: bytes32_to_vec(&contract.state_root), - tx_pointer: Some(proto_tx_pointer(&contract.tx_pointer)), - contract_id: contract.contract_id.as_ref().to_vec(), - } -} - -fn proto_contract_output_from_contract( - contract: &fuel_core_types::fuel_tx::output::contract::Contract, -) -> ProtoContractOutput { - ProtoContractOutput { - input_index: u32::from(contract.input_index), - balance_root: bytes32_to_vec(&contract.balance_root), - state_root: bytes32_to_vec(&contract.state_root), - } -} - -fn proto_output_from_output(output: &Output) -> ProtoOutput { - let variant = match output { - Output::Coin { - to, - amount, - asset_id, - } => ProtoOutputVariant::Coin(ProtoCoinOutput { - to: to.as_ref().to_vec(), - amount: *amount, - asset_id: asset_id.as_ref().to_vec(), - }), - Output::Contract(contract) => { - ProtoOutputVariant::Contract(proto_contract_output_from_contract(&contract)) - } - Output::Change { - to, - amount, - asset_id, - } => ProtoOutputVariant::Change(ProtoChangeOutput { - to: to.as_ref().to_vec(), - amount: *amount, - asset_id: asset_id.as_ref().to_vec(), - }), - Output::Variable { - to, - amount, - asset_id, - } => ProtoOutputVariant::Variable(ProtoVariableOutput { - to: to.as_ref().to_vec(), - amount: *amount, - asset_id: asset_id.as_ref().to_vec(), - }), - Output::ContractCreated { - contract_id, - state_root, - } => ProtoOutputVariant::ContractCreated(ProtoContractCreatedOutput { - contract_id: contract_id.as_ref().to_vec(), - state_root: bytes32_to_vec(&state_root), - }), - }; - - ProtoOutput { - variant: Some(variant), - } -} - -fn proto_upgrade_purpose(purpose: &UpgradePurpose) -> ProtoUpgradePurpose { - let variant = match purpose { - UpgradePurpose::ConsensusParameters { - witness_index, - checksum, - } => ProtoUpgradePurposeVariant::ConsensusParameters( - ProtoUpgradeConsensusParameters { - witness_index: u32::from(*witness_index), - checksum: checksum.as_ref().to_vec(), - }, - ), - UpgradePurpose::StateTransition { root } => { - ProtoUpgradePurposeVariant::StateTransition(ProtoUpgradeStateTransition { - root: root.as_ref().to_vec(), - }) - } - }; - - ProtoUpgradePurpose { - variant: Some(variant), - } -} - -fn proto_policies_from_policies( - policies: &fuel_core_types::fuel_tx::policies::Policies, -) -> ProtoPolicies { - let mut values = [0u64; 6]; - let mut truncated_len = 0; - if let Some(value) = policies.get(PolicyType::Tip) { - values[0] = value; - truncated_len = 1; - } - if let Some(value) = policies.get(PolicyType::WitnessLimit) { - values[1] = value; - truncated_len = 2; - } - if let Some(value) = policies.get(PolicyType::Maturity) { - values[2] = value; - truncated_len = 3; - } - if let Some(value) = policies.get(PolicyType::MaxFee) { - values[3] = value; - truncated_len = 4; - } - if let Some(value) = policies.get(PolicyType::Expiration) { - values[4] = value; - truncated_len = 5; - } - if let Some(value) = policies.get(PolicyType::Owner) { - values[5] = value; - truncated_len = 6; - } - let bits = policies.bits(); - values[..truncated_len].to_vec(); - ProtoPolicies { - bits, - values: values.to_vec(), - } -} - -fn tx_pointer_from_proto(proto: &ProtoTxPointer) -> Result { - let block_height = proto.block_height.into(); - #[allow(clippy::useless_conversion)] - let tx_index = proto.tx_index.try_into().map_err(|e| { - Error::Serialization(anyhow!("Could not convert tx_index to target type: {}", e)) - })?; - Ok(TxPointer::new(block_height, tx_index)) -} - -fn storage_slot_from_proto(proto: &ProtoStorageSlot) -> Result { - let key = Bytes32::try_from(proto.key.as_slice()).map_err(|e| { - Error::Serialization(anyhow!( - "Could not convert storage slot key to Bytes32: {}", - e - )) - })?; - let value = Bytes32::try_from(proto.value.as_slice()).map_err(|e| { - Error::Serialization(anyhow!( - "Could not convert storage slot value to Bytes32: {}", - e - )) - })?; - Ok(StorageSlot::new(key, value)) -} - -fn contract_input_from_proto( - proto: &ProtoContractInput, -) -> Result { - let utxo_proto = proto.utxo_id.as_ref().ok_or_else(|| { - Error::Serialization(anyhow!("Missing utxo_id on contract input")) - })?; - let utxo_id = utxo_id_from_proto(utxo_proto)?; - let balance_root = Bytes32::try_from(proto.balance_root.as_slice()).map_err(|e| { - Error::Serialization(anyhow!("Could not convert balance_root to Bytes32: {}", e)) - })?; - let state_root = Bytes32::try_from(proto.state_root.as_slice()).map_err(|e| { - Error::Serialization(anyhow!("Could not convert state_root to Bytes32: {}", e)) - })?; - let tx_pointer_proto = proto.tx_pointer.as_ref().ok_or_else(|| { - Error::Serialization(anyhow!("Missing tx_pointer on contract input")) - })?; - let tx_pointer = tx_pointer_from_proto(tx_pointer_proto)?; - let contract_id = - fuel_core_types::fuel_types::ContractId::try_from(proto.contract_id.as_slice()) - .map_err(|e| Error::Serialization(anyhow!(e)))?; - - Ok(fuel_core_types::fuel_tx::input::contract::Contract { - utxo_id, - balance_root, - state_root, - tx_pointer, - contract_id, - }) -} - -fn contract_output_from_proto( - proto: &ProtoContractOutput, -) -> Result { - let input_index = u16::try_from(proto.input_index).map_err(|e| { - Error::Serialization(anyhow!( - "Could not convert contract output input_index to u16: {}", - e - )) - })?; - let balance_root = Bytes32::try_from(proto.balance_root.as_slice()).map_err(|e| { - Error::Serialization(anyhow!( - "Could not convert contract output balance_root to Bytes32: {}", - e - )) - })?; - let state_root = Bytes32::try_from(proto.state_root.as_slice()).map_err(|e| { - Error::Serialization(anyhow!( - "Could not convert contract output state_root to Bytes32: {}", - e - )) - })?; - - Ok(fuel_core_types::fuel_tx::output::contract::Contract { - input_index, - balance_root, - state_root, - }) -} - -fn output_from_proto_output(proto_output: &ProtoOutput) -> Result { - match proto_output - .variant - .as_ref() - .ok_or_else(|| Error::Serialization(anyhow!("Missing output variant")))? - { - ProtoOutputVariant::Coin(coin) => { - let to = Address::try_from(coin.to.as_slice()) - .map_err(|e| Error::Serialization(anyhow!(e)))?; - let asset_id = - fuel_core_types::fuel_types::AssetId::try_from(coin.asset_id.as_slice()) - .map_err(|e| Error::Serialization(anyhow!(e)))?; - Ok(Output::coin(to, coin.amount, asset_id)) - } - ProtoOutputVariant::Contract(contract) => { - let contract = contract_output_from_proto(contract)?; - Ok(Output::Contract(contract)) - } - ProtoOutputVariant::Change(change) => { - let to = Address::try_from(change.to.as_slice()) - .map_err(|e| Error::Serialization(anyhow!(e)))?; - let asset_id = fuel_core_types::fuel_types::AssetId::try_from( - change.asset_id.as_slice(), - ) - .map_err(|e| Error::Serialization(anyhow!(e)))?; - Ok(Output::change(to, change.amount, asset_id)) - } - ProtoOutputVariant::Variable(variable) => { - let to = Address::try_from(variable.to.as_slice()) - .map_err(|e| Error::Serialization(anyhow!(e)))?; - let asset_id = fuel_core_types::fuel_types::AssetId::try_from( - variable.asset_id.as_slice(), - ) - .map_err(|e| Error::Serialization(anyhow!(e)))?; - Ok(Output::variable(to, variable.amount, asset_id)) - } - ProtoOutputVariant::ContractCreated(contract_created) => { - let contract_id = fuel_core_types::fuel_types::ContractId::try_from( - contract_created.contract_id.as_slice(), - ) - .map_err(|e| Error::Serialization(anyhow!(e)))?; - let state_root = Bytes32::try_from(contract_created.state_root.as_slice()) - .map_err(|e| { - Error::Serialization(anyhow!( - "Could not convert state_root to Bytes32: {}", - e - )) - })?; - Ok(Output::contract_created(contract_id, state_root)) - } - } -} - -fn upgrade_purpose_from_proto(proto: &ProtoUpgradePurpose) -> Result { - match proto - .variant - .as_ref() - .ok_or_else(|| Error::Serialization(anyhow!("Missing upgrade purpose variant")))? - { - ProtoUpgradePurposeVariant::ConsensusParameters(consensus) => { - let witness_index = u16::try_from(consensus.witness_index).map_err(|e| { - Error::Serialization(anyhow!( - "Could not convert witness_index to u16: {}", - e - )) - })?; - let checksum = - Bytes32::try_from(consensus.checksum.as_slice()).map_err(|e| { - Error::Serialization(anyhow!( - "Could not convert checksum to Bytes32: {}", - e - )) - })?; - Ok(UpgradePurpose::ConsensusParameters { - witness_index, - checksum, - }) - } - ProtoUpgradePurposeVariant::StateTransition(state) => { - let root = Bytes32::try_from(state.root.as_slice()).map_err(|e| { - Error::Serialization(anyhow!( - "Could not convert state transition root to Bytes32: {}", - e - )) - })?; - Ok(UpgradePurpose::StateTransition { root }) - } - } -} - -fn utxo_id_from_proto(proto_utxo: &ProtoUtxoId) -> Result { - let tx_id = Bytes32::try_from(proto_utxo.tx_id.as_slice()).map_err(|e| { - Error::Serialization(anyhow!("Could not convert tx_id to Bytes32: {}", e)) - })?; - let output_index = u16::try_from(proto_utxo.output_index).map_err(|e| { - Error::Serialization(anyhow!("Could not convert output_index to u16: {}", e)) - })?; - Ok(UtxoId::new(tx_id, output_index)) -} - -fn bytes32_to_vec(bytes: &fuel_core_types::fuel_types::Bytes32) -> Vec { - bytes.as_ref().to_vec() -} - -pub fn fuel_block_from_protobuf( - proto_block: ProtoBlock, - msg_ids: &[fuel_core_types::fuel_tx::MessageId], - event_inbox_root: Bytes32, -) -> Result { - let versioned_block = proto_block - .versioned_block - .ok_or_else(|| anyhow::anyhow!("Missing protobuf versioned_block")) - .map_err(Error::Serialization)?; - let partial_header = match &versioned_block { - ProtoVersionedBlock::V1(v1_block) => { - let proto_header = v1_block - .header - .clone() - .ok_or_else(|| anyhow::anyhow!("Missing protobuf header")) - .map_err(Error::Serialization)?; - partial_header_from_proto_header(&proto_header)? - } - }; - let txs = match versioned_block { - ProtoVersionedBlock::V1(v1_inner) => v1_inner - .transactions - .iter() - .map(tx_from_proto_tx) - .collect::>()?, - }; - FuelBlock::new( - partial_header, - txs, - msg_ids, - event_inbox_root, - #[cfg(feature = "fault-proving")] - &ChainId::default(), - ) - .map_err(|e| anyhow!(e)) - .map_err(Error::Serialization) -} - -pub fn partial_header_from_proto_header( - proto_header: &ProtoHeader, -) -> Result { - let partial_header = PartialBlockHeader { - consensus: proto_header_to_empty_consensus_header(&proto_header)?, - application: proto_header_to_empty_application_header(&proto_header)?, - }; - Ok(partial_header) -} - -pub fn tx_from_proto_tx(proto_tx: &ProtoTransaction) -> Result { - let variant = proto_tx - .variant - .as_ref() - .ok_or_else(|| Error::Serialization(anyhow!("Missing transaction variant")))?; - - match variant { - ProtoTransactionVariant::Script(proto_script) => { - let policies = proto_script - .policies - .clone() - .map(|p| policies_from_proto_policies(&p)) - .unwrap_or_default(); - let inputs = proto_script - .inputs - .iter() - .map(input_from_proto_input) - .collect::>>()?; - let outputs = proto_script - .outputs - .iter() - .map(output_from_proto_output) - .collect::>>()?; - let witnesses = proto_script - .witnesses - .iter() - .map(|w| Ok(Witness::from(w.clone()))) - .collect::>>()?; - let mut script_tx = FuelTransaction::script( - proto_script.script_gas_limit, - proto_script.script.clone(), - proto_script.script_data.clone(), - policies, - inputs, - outputs, - witnesses, - ); - *script_tx.receipts_root_mut() = Bytes32::try_from( - proto_script.receipts_root.as_slice(), - ) - .map_err(|e| { - Error::Serialization(anyhow!( - "Could not convert receipts_root to Bytes32: {}", - e - )) - })?; - - Ok(FuelTransaction::Script(script_tx)) - } - ProtoTransactionVariant::Create(proto_create) => { - let policies = proto_create - .policies - .clone() - .map(|p| policies_from_proto_policies(&p)) - .unwrap_or_default(); - let inputs = proto_create - .inputs - .iter() - .map(input_from_proto_input) - .collect::>>()?; - let outputs = proto_create - .outputs - .iter() - .map(output_from_proto_output) - .collect::>>()?; - let witnesses = proto_create - .witnesses - .iter() - .map(|w| Ok(Witness::from(w.clone()))) - .collect::>>()?; - let storage_slots = proto_create - .storage_slots - .iter() - .map(storage_slot_from_proto) - .collect::>>()?; - let salt = - fuel_core_types::fuel_types::Salt::try_from(proto_create.salt.as_slice()) - .map_err(|e| Error::Serialization(anyhow!(e)))?; - let bytecode_witness_index = - u16::try_from(proto_create.bytecode_witness_index).map_err(|e| { - Error::Serialization(anyhow!( - "Could not convert bytecode_witness_index to u16: {}", - e - )) - })?; - - let create_tx = FuelTransaction::create( - bytecode_witness_index, - policies, - salt, - storage_slots, - inputs, - outputs, - witnesses, - ); - - Ok(FuelTransaction::Create(create_tx)) - } - ProtoTransactionVariant::Mint(proto_mint) => { - let tx_pointer_proto = proto_mint.tx_pointer.as_ref().ok_or_else(|| { - Error::Serialization(anyhow!("Missing tx_pointer on mint transaction")) - })?; - let tx_pointer = tx_pointer_from_proto(tx_pointer_proto)?; - let input_contract_proto = - proto_mint.input_contract.as_ref().ok_or_else(|| { - Error::Serialization(anyhow!( - "Missing input_contract on mint transaction" - )) - })?; - let input_contract = contract_input_from_proto(input_contract_proto)?; - let output_contract_proto = - proto_mint.output_contract.as_ref().ok_or_else(|| { - Error::Serialization(anyhow!( - "Missing output_contract on mint transaction" - )) - })?; - let output_contract = contract_output_from_proto(output_contract_proto)?; - let mint_asset_id = fuel_core_types::fuel_types::AssetId::try_from( - proto_mint.mint_asset_id.as_slice(), - ) - .map_err(|e| Error::Serialization(anyhow!(e)))?; - - let mint_tx = FuelTransaction::mint( - tx_pointer, - input_contract, - output_contract, - proto_mint.mint_amount, - mint_asset_id, - proto_mint.gas_price, - ); - - Ok(FuelTransaction::Mint(mint_tx)) - } - ProtoTransactionVariant::Upgrade(proto_upgrade) => { - let purpose_proto = proto_upgrade.purpose.as_ref().ok_or_else(|| { - Error::Serialization(anyhow!("Missing purpose on upgrade transaction")) - })?; - let upgrade_purpose = upgrade_purpose_from_proto(purpose_proto)?; - let policies = proto_upgrade - .policies - .clone() - .map(|p| policies_from_proto_policies(&p)) - .unwrap_or_default(); - let inputs = proto_upgrade - .inputs - .iter() - .map(input_from_proto_input) - .collect::>>()?; - let outputs = proto_upgrade - .outputs - .iter() - .map(output_from_proto_output) - .collect::>>()?; - let witnesses = proto_upgrade - .witnesses - .iter() - .map(|w| Ok(Witness::from(w.clone()))) - .collect::>>()?; - - let upgrade_tx = FuelTransaction::upgrade( - upgrade_purpose, - policies, - inputs, - outputs, - witnesses, - ); - - Ok(FuelTransaction::Upgrade(upgrade_tx)) - } - ProtoTransactionVariant::Upload(proto_upload) => { - let policies = proto_upload - .policies - .clone() - .map(|p| policies_from_proto_policies(&p)) - .unwrap_or_default(); - let inputs = proto_upload - .inputs - .iter() - .map(input_from_proto_input) - .collect::>>()?; - let outputs = proto_upload - .outputs - .iter() - .map(output_from_proto_output) - .collect::>>()?; - let witnesses = proto_upload - .witnesses - .iter() - .map(|w| Ok(Witness::from(w.clone()))) - .collect::>>()?; - let root = Bytes32::try_from(proto_upload.root.as_slice()).map_err(|e| { - Error::Serialization(anyhow!( - "Could not convert upload root to Bytes32: {}", - e - )) - })?; - let witness_index = - u16::try_from(proto_upload.witness_index).map_err(|e| { - Error::Serialization(anyhow!( - "Could not convert witness_index to u16: {}", - e - )) - })?; - let subsection_index = - u16::try_from(proto_upload.subsection_index).map_err(|e| { - Error::Serialization(anyhow!( - "Could not convert subsection_index to u16: {}", - e - )) - })?; - let subsections_number = u16::try_from(proto_upload.subsections_number) - .map_err(|e| { - Error::Serialization(anyhow!( - "Could not convert subsections_number to u16: {}", - e - )) - })?; - let proof_set = proto_upload - .proof_set - .iter() - .map(|entry| { - Bytes32::try_from(entry.as_slice()).map_err(|e| { - Error::Serialization(anyhow!( - "Could not convert proof_set entry to Bytes32: {}", - e - )) - }) - }) - .collect::>>()?; - - let body = UploadBody { - root, - witness_index, - subsection_index, - subsections_number, - proof_set, - }; - - let upload_tx = - FuelTransaction::upload(body, policies, inputs, outputs, witnesses); - - Ok(FuelTransaction::Upload(upload_tx)) - } - ProtoTransactionVariant::Blob(proto_blob) => { - let policies = proto_blob - .policies - .clone() - .map(|p| policies_from_proto_policies(&p)) - .unwrap_or_default(); - let inputs = proto_blob - .inputs - .iter() - .map(input_from_proto_input) - .collect::>>()?; - let outputs = proto_blob - .outputs - .iter() - .map(output_from_proto_output) - .collect::>>()?; - let witnesses = proto_blob - .witnesses - .iter() - .map(|w| Ok(Witness::from(w.clone()))) - .collect::>>()?; - let blob_id = fuel_core_types::fuel_types::BlobId::try_from( - proto_blob.blob_id.as_slice(), - ) - .map_err(|e| Error::Serialization(anyhow!(e)))?; - let witness_index = u16::try_from(proto_blob.witness_index).map_err(|e| { - Error::Serialization(anyhow!( - "Could not convert blob witness_index to u16: {}", - e - )) - })?; - let body = BlobBody { - id: blob_id, - witness_index, - }; - - let blob_tx = - FuelTransaction::blob(body, policies, inputs, outputs, witnesses); - - Ok(FuelTransaction::Blob(blob_tx)) - } - } -} - -fn input_from_proto_input(proto_input: &ProtoInput) -> Result { - let variant = proto_input - .variant - .as_ref() - .ok_or_else(|| Error::Serialization(anyhow!("Missing input variant")))?; - - match variant { - ProtoInputVariant::CoinSigned(proto_coin_signed) => { - let utxo_proto = proto_coin_signed - .utxo_id - .as_ref() - .ok_or_else(|| Error::Serialization(anyhow!("Missing utxo_id")))?; - let utxo_id = utxo_id_from_proto(utxo_proto)?; - let owner = - Address::try_from(proto_coin_signed.owner.as_slice()).map_err(|e| { - Error::Serialization(anyhow!( - "Could not convert owner to Address: {}", - e - )) - })?; - let asset_id = fuel_core_types::fuel_types::AssetId::try_from( - proto_coin_signed.asset_id.as_slice(), - ) - .map_err(|e| Error::Serialization(anyhow!(e)))?; - let tx_pointer_proto = proto_coin_signed - .tx_pointer - .as_ref() - .ok_or_else(|| Error::Serialization(anyhow!("Missing tx_pointer")))?; - let tx_pointer = tx_pointer_from_proto(tx_pointer_proto)?; - let witness_index = - u16::try_from(proto_coin_signed.witness_index).map_err(|e| { - Error::Serialization(anyhow!( - "Could not convert witness_index to u16: {}", - e - )) - })?; - - Ok(Input::coin_signed( - utxo_id, - owner, - proto_coin_signed.amount, - asset_id, - tx_pointer, - witness_index, - )) - } - ProtoInputVariant::CoinPredicate(proto_coin_predicate) => { - let utxo_proto = proto_coin_predicate - .utxo_id - .as_ref() - .ok_or_else(|| Error::Serialization(anyhow!("Missing utxo_id")))?; - let utxo_id = utxo_id_from_proto(utxo_proto)?; - let owner = Address::try_from(proto_coin_predicate.owner.as_slice()) - .map_err(|e| { - Error::Serialization(anyhow!( - "Could not convert owner to Address: {}", - e - )) - })?; - let asset_id = fuel_core_types::fuel_types::AssetId::try_from( - proto_coin_predicate.asset_id.as_slice(), - ) - .map_err(|e| Error::Serialization(anyhow!(e)))?; - let tx_pointer_proto = proto_coin_predicate - .tx_pointer - .as_ref() - .ok_or_else(|| Error::Serialization(anyhow!("Missing tx_pointer")))?; - let tx_pointer = tx_pointer_from_proto(tx_pointer_proto)?; - - Ok(Input::coin_predicate( - utxo_id, - owner, - proto_coin_predicate.amount, - asset_id, - tx_pointer, - proto_coin_predicate.predicate_gas_used, - proto_coin_predicate.predicate.clone(), - proto_coin_predicate.predicate_data.clone(), - )) - } - ProtoInputVariant::Contract(proto_contract) => { - let contract = contract_input_from_proto(proto_contract)?; - Ok(Input::Contract(contract)) - } - ProtoInputVariant::MessageCoinSigned(proto_message) => { - let sender = - Address::try_from(proto_message.sender.as_slice()).map_err(|e| { - Error::Serialization(anyhow!( - "Could not convert sender to Address: {}", - e - )) - })?; - let recipient = Address::try_from(proto_message.recipient.as_slice()) - .map_err(|e| { - Error::Serialization(anyhow!( - "Could not convert recipient to Address: {}", - e - )) - })?; - let nonce = fuel_core_types::fuel_types::Nonce::try_from( - proto_message.nonce.as_slice(), - ) - .map_err(|e| Error::Serialization(anyhow!(e)))?; - let witness_index = - u16::try_from(proto_message.witness_index).map_err(|e| { - Error::Serialization(anyhow!( - "Could not convert witness_index to u16: {}", - e - )) - })?; - - Ok(Input::message_coin_signed( - sender, - recipient, - proto_message.amount, - nonce, - witness_index, - )) - } - ProtoInputVariant::MessageCoinPredicate(proto_message) => { - let sender = - Address::try_from(proto_message.sender.as_slice()).map_err(|e| { - Error::Serialization(anyhow!( - "Could not convert sender to Address: {}", - e - )) - })?; - let recipient = Address::try_from(proto_message.recipient.as_slice()) - .map_err(|e| { - Error::Serialization(anyhow!( - "Could not convert recipient to Address: {}", - e - )) - })?; - let nonce = fuel_core_types::fuel_types::Nonce::try_from( - proto_message.nonce.as_slice(), - ) - .map_err(|e| Error::Serialization(anyhow!(e)))?; - - Ok(Input::message_coin_predicate( - sender, - recipient, - proto_message.amount, - nonce, - proto_message.predicate_gas_used, - proto_message.predicate.clone(), - proto_message.predicate_data.clone(), - )) - } - ProtoInputVariant::MessageDataSigned(proto_message) => { - let sender = - Address::try_from(proto_message.sender.as_slice()).map_err(|e| { - Error::Serialization(anyhow!( - "Could not convert sender to Address: {}", - e - )) - })?; - let recipient = Address::try_from(proto_message.recipient.as_slice()) - .map_err(|e| { - Error::Serialization(anyhow!( - "Could not convert recipient to Address: {}", - e - )) - })?; - let nonce = fuel_core_types::fuel_types::Nonce::try_from( - proto_message.nonce.as_slice(), - ) - .map_err(|e| Error::Serialization(anyhow!(e)))?; - let witness_index = - u16::try_from(proto_message.witness_index).map_err(|e| { - Error::Serialization(anyhow!( - "Could not convert witness_index to u16: {}", - e - )) - })?; - - Ok(Input::message_data_signed( - sender, - recipient, - proto_message.amount, - nonce, - witness_index, - proto_message.data.clone(), - )) - } - ProtoInputVariant::MessageDataPredicate(proto_message) => { - let sender = - Address::try_from(proto_message.sender.as_slice()).map_err(|e| { - Error::Serialization(anyhow!( - "Could not convert sender to Address: {}", - e - )) - })?; - let recipient = Address::try_from(proto_message.recipient.as_slice()) - .map_err(|e| { - Error::Serialization(anyhow!( - "Could not convert recipient to Address: {}", - e - )) - })?; - let nonce = fuel_core_types::fuel_types::Nonce::try_from( - proto_message.nonce.as_slice(), - ) - .map_err(|e| Error::Serialization(anyhow!(e)))?; - - Ok(Input::message_data_predicate( - sender, - recipient, - proto_message.amount, - nonce, - proto_message.predicate_gas_used, - proto_message.data.clone(), - proto_message.predicate.clone(), - proto_message.predicate_data.clone(), - )) - } - } -} - -fn policies_from_proto_policies(proto_policies: &ProtoPolicies) -> FuelPolicies { - let ProtoPolicies { bits, values } = proto_policies; - let mut policies = FuelPolicies::default(); - let bits = - PoliciesBits::from_bits(*bits).expect("Should be able to create from `u32`"); - if bits.contains(PoliciesBits::Tip) - && let Some(tip) = values.first() - { - policies.set(PolicyType::Tip, Some(*tip)); - } - if bits.contains(PoliciesBits::WitnessLimit) - && let Some(witness_limit) = values.get(1) - { - policies.set(PolicyType::WitnessLimit, Some(*witness_limit)); - } - if bits.contains(PoliciesBits::Maturity) - && let Some(maturity) = values.get(2) - { - policies.set(PolicyType::Maturity, Some(*maturity)); - } - if bits.contains(PoliciesBits::MaxFee) - && let Some(max_fee) = values.get(3) - { - policies.set(PolicyType::MaxFee, Some(*max_fee)); - } - if bits.contains(PoliciesBits::Expiration) - && let Some(expiration) = values.get(4) - { - policies.set(PolicyType::Expiration, Some(*expiration)); - } - if bits.contains(PoliciesBits::Owner) - && let Some(owner) = values.get(5) - { - policies.set(PolicyType::Owner, Some(*owner)); - } - policies -} - -pub fn proto_header_to_empty_application_header( - proto_header: &ProtoHeader, -) -> Result> { - match proto_header.versioned_header.clone() { - Some(ProtoVersionedHeader::V1(header)) => { - let app_header = ApplicationHeader { - da_height: DaBlockHeight::from(header.da_height), - consensus_parameters_version: header.consensus_parameters_version, - state_transition_bytecode_version: header - .state_transition_bytecode_version, - generated: Empty {}, - }; - Ok(app_header) - } - Some(ProtoVersionedHeader::V2(header)) => { - if cfg!(feature = "fault-proving") { - let app_header = ApplicationHeader { - da_height: DaBlockHeight::from(header.da_height), - consensus_parameters_version: header.consensus_parameters_version, - state_transition_bytecode_version: header - .state_transition_bytecode_version, - generated: Empty {}, - }; - Ok(app_header) - } else { - Err(anyhow!("V2 headers require the 'fault-proving' feature")) - .map_err(Error::Serialization) - } - } - None => Err(anyhow!("Missing protobuf versioned_header")) - .map_err(Error::Serialization), - } -} - -/// Alias the consensus header into an empty one. -pub fn proto_header_to_empty_consensus_header( - proto_header: &ProtoHeader, -) -> Result> { - match proto_header.versioned_header.clone() { - Some(ProtoVersionedHeader::V1(header)) => { - let consensus_header = ConsensusHeader { - prev_root: *Bytes32::from_bytes_ref_checked(&header.prev_root).ok_or( - Error::Serialization(anyhow!("Could create `Bytes32` from bytes")), - )?, - height: header.height.into(), - time: tai64::Tai64(header.time), - generated: Empty {}, - }; - Ok(consensus_header) - } - Some(ProtoVersionedHeader::V2(header)) => { - if cfg!(feature = "fault-proving") { - let consensus_header = ConsensusHeader { - prev_root: *Bytes32::from_bytes_ref_checked(&header.prev_root) - .ok_or(Error::Serialization(anyhow!( - "Could create `Bytes32` from bytes" - )))?, - height: header.height.into(), - time: tai64::Tai64(header.time), - generated: Empty {}, - }; - Ok(consensus_header) - } else { - Err(anyhow!("V2 headers require the 'fault-proving' feature")) - .map_err(Error::Serialization) - } - } - None => Err(anyhow!("Missing protobuf versioned_header")) - .map_err(Error::Serialization), - } -} +pub mod fuel_to_proto_conversions; +pub mod proto_to_fuel_conversions; // TODO: Add coverage for V2 Block stuff // https://github.com/FuelLabs/fuel-core/issues/3139 @@ -1480,6 +59,7 @@ mod tests { use super::*; use fuel_core_types::test_helpers::arb_block; use proptest::prelude::*; + use crate::blocks::importer_and_db_source::serializer_adapter::proto_to_fuel_conversions::fuel_block_from_protobuf; proptest! { #![proptest_config(ProptestConfig { diff --git a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/fuel_to_proto_conversions.rs b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/fuel_to_proto_conversions.rs new file mode 100644 index 00000000000..4b22aaaab11 --- /dev/null +++ b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/fuel_to_proto_conversions.rs @@ -0,0 +1,564 @@ +#[cfg(feature = "fault-proving")] +use crate::protobuf_types::V2Header as ProtoV2Header; +use crate::{ + blocks::importer_and_db_source::serializer_adapter::proto_to_fuel_conversions::bytes32_to_vec, + protobuf_types::{ + BlobTransaction as ProtoBlobTx, + ChangeOutput as ProtoChangeOutput, + CoinOutput as ProtoCoinOutput, + CoinPredicateInput as ProtoCoinPredicateInput, + CoinSignedInput as ProtoCoinSignedInput, + ContractCreatedOutput as ProtoContractCreatedOutput, + ContractInput as ProtoContractInput, + ContractOutput as ProtoContractOutput, + CreateTransaction as ProtoCreateTx, + Header as ProtoHeader, + Input as ProtoInput, + MessageCoinPredicateInput as ProtoMessageCoinPredicateInput, + MessageCoinSignedInput as ProtoMessageCoinSignedInput, + MessageDataPredicateInput as ProtoMessageDataPredicateInput, + MessageDataSignedInput as ProtoMessageDataSignedInput, + MintTransaction as ProtoMintTx, + Output as ProtoOutput, + Policies as ProtoPolicies, + ScriptTransaction as ProtoScriptTx, + StorageSlot as ProtoStorageSlot, + Transaction as ProtoTransaction, + TxPointer as ProtoTxPointer, + UpgradeConsensusParameters as ProtoUpgradeConsensusParameters, + UpgradePurpose as ProtoUpgradePurpose, + UpgradeStateTransition as ProtoUpgradeStateTransition, + UpgradeTransaction as ProtoUpgradeTx, + UploadTransaction as ProtoUploadTx, + UtxoId as ProtoUtxoId, + V1Header as ProtoV1Header, + VariableOutput as ProtoVariableOutput, + header::VersionedHeader as ProtoVersionedHeader, + input::Variant as ProtoInputVariant, + output::Variant as ProtoOutputVariant, + transaction::Variant as ProtoTransactionVariant, + upgrade_purpose::Variant as ProtoUpgradePurposeVariant, + }, +}; + +use fuel_core_types::{ + blockchain::{ + header::{ + BlockHeader, + BlockHeaderV1, + ConsensusHeader, + GeneratedConsensusFields, + }, + primitives::BlockId, + }, + fuel_tx::{ + Input, + Output, + StorageSlot, + Transaction as FuelTransaction, + TxPointer, + UpgradePurpose, + UtxoId, + field::{ + BlobId as _, + BytecodeRoot as _, + BytecodeWitnessIndex as _, + InputContract as _, + Inputs, + MintAmount as _, + MintAssetId as _, + MintGasPrice as _, + OutputContract as _, + Outputs, + Policies as _, + ProofSet as _, + ReceiptsRoot as _, + Salt as _, + Script as _, + ScriptData as _, + ScriptGasLimit as _, + StorageSlots as _, + SubsectionIndex as _, + SubsectionsNumber as _, + TxPointer as TxPointerField, + UpgradePurpose as UpgradePurposeField, + Witnesses as _, + }, + policies::PolicyType, + }, +}; + +pub fn proto_header_from_header(header: &BlockHeader) -> ProtoHeader { + let block_id = header.id(); + let consensus = header.consensus(); + let versioned_header = match header { + BlockHeader::V1(header) => { + let proto_v1_header = + proto_v1_header_from_v1_header(&consensus, &block_id, header); + ProtoVersionedHeader::V1(proto_v1_header) + } + #[cfg(feature = "fault-proving")] + BlockHeader::V2(header) => { + let proto_v2_header = + proto_v2_header_from_v2_header(consensus, &block_id, header); + ProtoVersionedHeader::V2(proto_v2_header) + } + }; + + ProtoHeader { + versioned_header: Some(versioned_header), + } +} + +fn proto_v1_header_from_v1_header( + consensus: &ConsensusHeader, + block_id: &BlockId, + header: &BlockHeaderV1, +) -> ProtoV1Header { + let application = header.application(); + let generated = application.generated; + + ProtoV1Header { + da_height: application.da_height.0, + consensus_parameters_version: application.consensus_parameters_version, + state_transition_bytecode_version: application.state_transition_bytecode_version, + transactions_count: u32::from(generated.transactions_count), + message_receipt_count: generated.message_receipt_count, + transactions_root: bytes32_to_vec(&generated.transactions_root), + message_outbox_root: bytes32_to_vec(&generated.message_outbox_root), + event_inbox_root: bytes32_to_vec(&generated.event_inbox_root), + prev_root: bytes32_to_vec(&consensus.prev_root), + height: u32::from(consensus.height), + time: consensus.time.0, + application_hash: bytes32_to_vec(&consensus.generated.application_hash), + block_id: Some(block_id.as_slice().to_vec()), + } +} + +#[cfg(feature = "fault-proving")] +fn proto_v2_header_from_v2_header( + consensus: &ConsensusHeader, + block_id: &BlockId, + header: &BlockHeaderV2, +) -> ProtoV2Header { + let application = *header.application(); + let generated = application.generated; + + ProtoV2Header { + da_height: application.da_height.0, + consensus_parameters_version: application.consensus_parameters_version, + state_transition_bytecode_version: application.state_transition_bytecode_version, + transactions_count: u32::from(generated.transactions_count), + message_receipt_count: generated.message_receipt_count, + transactions_root: bytes32_to_vec(&generated.transactions_root), + message_outbox_root: bytes32_to_vec(&generated.message_outbox_root), + event_inbox_root: bytes32_to_vec(&generated.event_inbox_root), + tx_id_commitment: bytes32_to_vec(&generated.tx_id_commitment), + prev_root: bytes32_to_vec(&consensus.prev_root), + height: u32::from(consensus.height), + time: consensus.time.0, + application_hash: bytes32_to_vec(&consensus.generated.application_hash), + block_id: Some(block_id.as_slice().to_vec()), + } +} + +pub fn proto_tx_from_tx(tx: &FuelTransaction) -> ProtoTransaction { + match tx { + FuelTransaction::Script(script) => { + let proto_script = ProtoScriptTx { + script_gas_limit: *script.script_gas_limit(), + receipts_root: bytes32_to_vec(script.receipts_root()), + script: script.script().clone(), + script_data: script.script_data().clone(), + policies: Some(proto_policies_from_policies(script.policies())), + inputs: script.inputs().iter().map(proto_input_from_input).collect(), + outputs: script + .outputs() + .iter() + .map(proto_output_from_output) + .collect(), + witnesses: script + .witnesses() + .iter() + .map(|witness| witness.as_ref().to_vec()) + .collect(), + metadata: None, + }; + + ProtoTransaction { + variant: Some(ProtoTransactionVariant::Script(proto_script)), + } + } + FuelTransaction::Create(create) => { + let proto_create = ProtoCreateTx { + bytecode_witness_index: u32::from(*create.bytecode_witness_index()), + salt: create.salt().as_ref().to_vec(), + storage_slots: create + .storage_slots() + .iter() + .map(proto_storage_slot_from_storage_slot) + .collect(), + policies: Some(proto_policies_from_policies(create.policies())), + inputs: create.inputs().iter().map(proto_input_from_input).collect(), + outputs: create + .outputs() + .iter() + .map(proto_output_from_output) + .collect(), + witnesses: create + .witnesses() + .iter() + .map(|witness| witness.as_ref().to_vec()) + .collect(), + metadata: None, + }; + + ProtoTransaction { + variant: Some(ProtoTransactionVariant::Create(proto_create)), + } + } + FuelTransaction::Mint(mint) => { + let proto_mint = ProtoMintTx { + tx_pointer: Some(proto_tx_pointer(mint.tx_pointer())), + input_contract: Some(proto_contract_input_from_contract( + mint.input_contract(), + )), + output_contract: Some(proto_contract_output_from_contract( + mint.output_contract(), + )), + mint_amount: *mint.mint_amount(), + mint_asset_id: mint.mint_asset_id().as_ref().to_vec(), + gas_price: *mint.gas_price(), + metadata: None, + }; + + ProtoTransaction { + variant: Some(ProtoTransactionVariant::Mint(proto_mint)), + } + } + FuelTransaction::Upgrade(upgrade) => { + let proto_upgrade = ProtoUpgradeTx { + purpose: Some(proto_upgrade_purpose(upgrade.upgrade_purpose())), + policies: Some(proto_policies_from_policies(upgrade.policies())), + inputs: upgrade + .inputs() + .iter() + .map(proto_input_from_input) + .collect(), + outputs: upgrade + .outputs() + .iter() + .map(proto_output_from_output) + .collect(), + witnesses: upgrade + .witnesses() + .iter() + .map(|witness| witness.as_ref().to_vec()) + .collect(), + metadata: None, + }; + + ProtoTransaction { + variant: Some(ProtoTransactionVariant::Upgrade(proto_upgrade)), + } + } + FuelTransaction::Upload(upload) => { + let proto_upload = ProtoUploadTx { + root: bytes32_to_vec(upload.bytecode_root()), + witness_index: u32::from(*upload.bytecode_witness_index()), + subsection_index: u32::from(*upload.subsection_index()), + subsections_number: u32::from(*upload.subsections_number()), + proof_set: upload.proof_set().iter().map(bytes32_to_vec).collect(), + policies: Some(proto_policies_from_policies(upload.policies())), + inputs: upload.inputs().iter().map(proto_input_from_input).collect(), + outputs: upload + .outputs() + .iter() + .map(proto_output_from_output) + .collect(), + witnesses: upload + .witnesses() + .iter() + .map(|witness| witness.as_ref().to_vec()) + .collect(), + metadata: None, + }; + + ProtoTransaction { + variant: Some(ProtoTransactionVariant::Upload(proto_upload)), + } + } + FuelTransaction::Blob(blob) => { + let proto_blob = ProtoBlobTx { + blob_id: blob.blob_id().as_ref().to_vec(), + witness_index: u32::from(*blob.bytecode_witness_index()), + policies: Some(proto_policies_from_policies(blob.policies())), + inputs: blob.inputs().iter().map(proto_input_from_input).collect(), + outputs: blob + .outputs() + .iter() + .map(proto_output_from_output) + .collect(), + witnesses: blob + .witnesses() + .iter() + .map(|witness| witness.as_ref().to_vec()) + .collect(), + metadata: None, + }; + + ProtoTransaction { + variant: Some(ProtoTransactionVariant::Blob(proto_blob)), + } + } + } +} + +fn proto_input_from_input(input: &Input) -> ProtoInput { + match input { + Input::CoinSigned(coin_signed) => ProtoInput { + variant: Some(ProtoInputVariant::CoinSigned(ProtoCoinSignedInput { + utxo_id: Some(proto_utxo_id_from_utxo_id(&coin_signed.utxo_id)), + owner: coin_signed.owner.as_ref().to_vec(), + amount: coin_signed.amount, + asset_id: coin_signed.asset_id.as_ref().to_vec(), + tx_pointer: Some(proto_tx_pointer(&coin_signed.tx_pointer)), + witness_index: coin_signed.witness_index.into(), + predicate_gas_used: 0, + predicate: vec![], + predicate_data: vec![], + })), + }, + Input::CoinPredicate(coin_predicate) => ProtoInput { + variant: Some(ProtoInputVariant::CoinPredicate(ProtoCoinPredicateInput { + utxo_id: Some(proto_utxo_id_from_utxo_id(&coin_predicate.utxo_id)), + owner: coin_predicate.owner.as_ref().to_vec(), + amount: coin_predicate.amount, + asset_id: coin_predicate.asset_id.as_ref().to_vec(), + tx_pointer: Some(proto_tx_pointer(&coin_predicate.tx_pointer)), + witness_index: 0, + predicate_gas_used: coin_predicate.predicate_gas_used, + predicate: coin_predicate.predicate.as_ref().to_vec(), + predicate_data: coin_predicate.predicate_data.as_ref().to_vec(), + })), + }, + Input::Contract(contract) => ProtoInput { + variant: Some(ProtoInputVariant::Contract(ProtoContractInput { + utxo_id: Some(proto_utxo_id_from_utxo_id(&contract.utxo_id)), + balance_root: bytes32_to_vec(&contract.balance_root), + state_root: bytes32_to_vec(&contract.state_root), + tx_pointer: Some(proto_tx_pointer(&contract.tx_pointer)), + contract_id: contract.contract_id.as_ref().to_vec(), + })), + }, + Input::MessageCoinSigned(message) => ProtoInput { + variant: Some(ProtoInputVariant::MessageCoinSigned( + ProtoMessageCoinSignedInput { + sender: message.sender.as_ref().to_vec(), + recipient: message.recipient.as_ref().to_vec(), + amount: message.amount, + nonce: message.nonce.as_ref().to_vec(), + witness_index: message.witness_index.into(), + predicate_gas_used: 0, + data: Vec::new(), + predicate: Vec::new(), + predicate_data: Vec::new(), + }, + )), + }, + Input::MessageCoinPredicate(message) => ProtoInput { + variant: Some(ProtoInputVariant::MessageCoinPredicate( + ProtoMessageCoinPredicateInput { + sender: message.sender.as_ref().to_vec(), + recipient: message.recipient.as_ref().to_vec(), + amount: message.amount, + nonce: message.nonce.as_ref().to_vec(), + witness_index: 0, + predicate_gas_used: message.predicate_gas_used, + data: Vec::new(), + predicate: message.predicate.as_ref().to_vec(), + predicate_data: message.predicate_data.as_ref().to_vec(), + }, + )), + }, + Input::MessageDataSigned(message) => ProtoInput { + variant: Some(ProtoInputVariant::MessageDataSigned( + ProtoMessageDataSignedInput { + sender: message.sender.as_ref().to_vec(), + recipient: message.recipient.as_ref().to_vec(), + amount: message.amount, + nonce: message.nonce.as_ref().to_vec(), + witness_index: message.witness_index.into(), + predicate_gas_used: 0, + data: message.data.as_ref().to_vec(), + predicate: Vec::new(), + predicate_data: Vec::new(), + }, + )), + }, + Input::MessageDataPredicate(message) => ProtoInput { + variant: Some(ProtoInputVariant::MessageDataPredicate( + ProtoMessageDataPredicateInput { + sender: message.sender.as_ref().to_vec(), + recipient: message.recipient.as_ref().to_vec(), + amount: message.amount, + nonce: message.nonce.as_ref().to_vec(), + witness_index: 0, + predicate_gas_used: message.predicate_gas_used, + data: message.data.as_ref().to_vec(), + predicate: message.predicate.as_ref().to_vec(), + predicate_data: message.predicate_data.as_ref().to_vec(), + }, + )), + }, + } +} + +fn proto_utxo_id_from_utxo_id(utxo_id: &UtxoId) -> ProtoUtxoId { + ProtoUtxoId { + tx_id: utxo_id.tx_id().as_ref().to_vec(), + output_index: utxo_id.output_index().into(), + } +} + +fn proto_tx_pointer(tx_pointer: &TxPointer) -> ProtoTxPointer { + ProtoTxPointer { + block_height: tx_pointer.block_height().into(), + tx_index: tx_pointer.tx_index().into(), + } +} + +fn proto_storage_slot_from_storage_slot(slot: &StorageSlot) -> ProtoStorageSlot { + ProtoStorageSlot { + key: slot.key().as_ref().to_vec(), + value: slot.value().as_ref().to_vec(), + } +} + +fn proto_contract_input_from_contract( + contract: &fuel_core_types::fuel_tx::input::contract::Contract, +) -> ProtoContractInput { + ProtoContractInput { + utxo_id: Some(proto_utxo_id_from_utxo_id(&contract.utxo_id)), + balance_root: bytes32_to_vec(&contract.balance_root), + state_root: bytes32_to_vec(&contract.state_root), + tx_pointer: Some(proto_tx_pointer(&contract.tx_pointer)), + contract_id: contract.contract_id.as_ref().to_vec(), + } +} + +fn proto_contract_output_from_contract( + contract: &fuel_core_types::fuel_tx::output::contract::Contract, +) -> ProtoContractOutput { + ProtoContractOutput { + input_index: u32::from(contract.input_index), + balance_root: bytes32_to_vec(&contract.balance_root), + state_root: bytes32_to_vec(&contract.state_root), + } +} + +fn proto_output_from_output(output: &Output) -> ProtoOutput { + let variant = match output { + Output::Coin { + to, + amount, + asset_id, + } => ProtoOutputVariant::Coin(ProtoCoinOutput { + to: to.as_ref().to_vec(), + amount: *amount, + asset_id: asset_id.as_ref().to_vec(), + }), + Output::Contract(contract) => { + ProtoOutputVariant::Contract(proto_contract_output_from_contract(&contract)) + } + Output::Change { + to, + amount, + asset_id, + } => ProtoOutputVariant::Change(ProtoChangeOutput { + to: to.as_ref().to_vec(), + amount: *amount, + asset_id: asset_id.as_ref().to_vec(), + }), + Output::Variable { + to, + amount, + asset_id, + } => ProtoOutputVariant::Variable(ProtoVariableOutput { + to: to.as_ref().to_vec(), + amount: *amount, + asset_id: asset_id.as_ref().to_vec(), + }), + Output::ContractCreated { + contract_id, + state_root, + } => ProtoOutputVariant::ContractCreated(ProtoContractCreatedOutput { + contract_id: contract_id.as_ref().to_vec(), + state_root: bytes32_to_vec(&state_root), + }), + }; + + ProtoOutput { + variant: Some(variant), + } +} + +fn proto_upgrade_purpose(purpose: &UpgradePurpose) -> ProtoUpgradePurpose { + let variant = match purpose { + UpgradePurpose::ConsensusParameters { + witness_index, + checksum, + } => ProtoUpgradePurposeVariant::ConsensusParameters( + ProtoUpgradeConsensusParameters { + witness_index: u32::from(*witness_index), + checksum: checksum.as_ref().to_vec(), + }, + ), + UpgradePurpose::StateTransition { root } => { + ProtoUpgradePurposeVariant::StateTransition(ProtoUpgradeStateTransition { + root: root.as_ref().to_vec(), + }) + } + }; + + ProtoUpgradePurpose { + variant: Some(variant), + } +} + +fn proto_policies_from_policies( + policies: &fuel_core_types::fuel_tx::policies::Policies, +) -> ProtoPolicies { + let mut values = [0u64; 6]; + let mut truncated_len = 0; + if let Some(value) = policies.get(PolicyType::Tip) { + values[0] = value; + truncated_len = 1; + } + if let Some(value) = policies.get(PolicyType::WitnessLimit) { + values[1] = value; + truncated_len = 2; + } + if let Some(value) = policies.get(PolicyType::Maturity) { + values[2] = value; + truncated_len = 3; + } + if let Some(value) = policies.get(PolicyType::MaxFee) { + values[3] = value; + truncated_len = 4; + } + if let Some(value) = policies.get(PolicyType::Expiration) { + values[4] = value; + truncated_len = 5; + } + if let Some(value) = policies.get(PolicyType::Owner) { + values[5] = value; + truncated_len = 6; + } + let bits = policies.bits(); + values[..truncated_len].to_vec(); + ProtoPolicies { + bits, + values: values.to_vec(), + } +} diff --git a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/proto_to_fuel_conversions.rs b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/proto_to_fuel_conversions.rs index e69de29bb2d..6aec4246413 100644 --- a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/proto_to_fuel_conversions.rs +++ b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/proto_to_fuel_conversions.rs @@ -0,0 +1,919 @@ +#[cfg(feature = "fault-proving")] +use crate::protobuf_types::V2Header as ProtoV2Header; +use crate::{ + protobuf_types::{ + Block as ProtoBlock, + ContractInput as ProtoContractInput, + ContractOutput as ProtoContractOutput, + Header as ProtoHeader, + Input as ProtoInput, + Output as ProtoOutput, + Policies as ProtoPolicies, + StorageSlot as ProtoStorageSlot, + Transaction as ProtoTransaction, + TxPointer as ProtoTxPointer, + UpgradePurpose as ProtoUpgradePurpose, + UtxoId as ProtoUtxoId, + block::VersionedBlock as ProtoVersionedBlock, + header::VersionedHeader as ProtoVersionedHeader, + input::Variant as ProtoInputVariant, + output::Variant as ProtoOutputVariant, + transaction::Variant as ProtoTransactionVariant, + upgrade_purpose::Variant as ProtoUpgradePurposeVariant, + }, + result::Error, +}; +use anyhow::anyhow; +use fuel_core_types::{ + blockchain::{ + block::Block as FuelBlock, + header::{ + ApplicationHeader, + ConsensusHeader, + PartialBlockHeader, + }, + primitives::{ + DaBlockHeight, + Empty, + }, + }, + fuel_tx::{ + Address, + BlobBody, + Bytes32, + Input, + Output, + StorageSlot, + Transaction as FuelTransaction, + TxPointer, + UpgradePurpose, + UploadBody, + UtxoId, + Witness, + field::ReceiptsRoot as _, + policies::{ + Policies as FuelPolicies, + PoliciesBits, + PolicyType, + }, + }, + tai64, +}; + +fn tx_pointer_from_proto(proto: &ProtoTxPointer) -> crate::result::Result { + let block_height = proto.block_height.into(); + #[allow(clippy::useless_conversion)] + let tx_index = proto.tx_index.try_into().map_err(|e| { + Error::Serialization(anyhow!("Could not convert tx_index to target type: {}", e)) + })?; + Ok(TxPointer::new(block_height, tx_index)) +} + +fn storage_slot_from_proto( + proto: &ProtoStorageSlot, +) -> crate::result::Result { + let key = Bytes32::try_from(proto.key.as_slice()).map_err(|e| { + Error::Serialization(anyhow!( + "Could not convert storage slot key to Bytes32: {}", + e + )) + })?; + let value = Bytes32::try_from(proto.value.as_slice()).map_err(|e| { + Error::Serialization(anyhow!( + "Could not convert storage slot value to Bytes32: {}", + e + )) + })?; + Ok(StorageSlot::new(key, value)) +} + +fn contract_input_from_proto( + proto: &ProtoContractInput, +) -> crate::result::Result { + let utxo_proto = proto.utxo_id.as_ref().ok_or_else(|| { + Error::Serialization(anyhow!("Missing utxo_id on contract input")) + })?; + let utxo_id = utxo_id_from_proto(utxo_proto)?; + let balance_root = Bytes32::try_from(proto.balance_root.as_slice()).map_err(|e| { + Error::Serialization(anyhow!("Could not convert balance_root to Bytes32: {}", e)) + })?; + let state_root = Bytes32::try_from(proto.state_root.as_slice()).map_err(|e| { + Error::Serialization(anyhow!("Could not convert state_root to Bytes32: {}", e)) + })?; + let tx_pointer_proto = proto.tx_pointer.as_ref().ok_or_else(|| { + Error::Serialization(anyhow!("Missing tx_pointer on contract input")) + })?; + let tx_pointer = tx_pointer_from_proto(tx_pointer_proto)?; + let contract_id = + fuel_core_types::fuel_types::ContractId::try_from(proto.contract_id.as_slice()) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + + Ok(fuel_core_types::fuel_tx::input::contract::Contract { + utxo_id, + balance_root, + state_root, + tx_pointer, + contract_id, + }) +} + +fn contract_output_from_proto( + proto: &ProtoContractOutput, +) -> crate::result::Result { + let input_index = u16::try_from(proto.input_index).map_err(|e| { + Error::Serialization(anyhow!( + "Could not convert contract output input_index to u16: {}", + e + )) + })?; + let balance_root = Bytes32::try_from(proto.balance_root.as_slice()).map_err(|e| { + Error::Serialization(anyhow!( + "Could not convert contract output balance_root to Bytes32: {}", + e + )) + })?; + let state_root = Bytes32::try_from(proto.state_root.as_slice()).map_err(|e| { + Error::Serialization(anyhow!( + "Could not convert contract output state_root to Bytes32: {}", + e + )) + })?; + + Ok(fuel_core_types::fuel_tx::output::contract::Contract { + input_index, + balance_root, + state_root, + }) +} + +fn output_from_proto_output(proto_output: &ProtoOutput) -> crate::result::Result { + match proto_output + .variant + .as_ref() + .ok_or_else(|| Error::Serialization(anyhow!("Missing output variant")))? + { + ProtoOutputVariant::Coin(coin) => { + let to = Address::try_from(coin.to.as_slice()) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + let asset_id = + fuel_core_types::fuel_types::AssetId::try_from(coin.asset_id.as_slice()) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + Ok(Output::coin(to, coin.amount, asset_id)) + } + ProtoOutputVariant::Contract(contract) => { + let contract = contract_output_from_proto(contract)?; + Ok(Output::Contract(contract)) + } + ProtoOutputVariant::Change(change) => { + let to = Address::try_from(change.to.as_slice()) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + let asset_id = fuel_core_types::fuel_types::AssetId::try_from( + change.asset_id.as_slice(), + ) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + Ok(Output::change(to, change.amount, asset_id)) + } + ProtoOutputVariant::Variable(variable) => { + let to = Address::try_from(variable.to.as_slice()) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + let asset_id = fuel_core_types::fuel_types::AssetId::try_from( + variable.asset_id.as_slice(), + ) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + Ok(Output::variable(to, variable.amount, asset_id)) + } + ProtoOutputVariant::ContractCreated(contract_created) => { + let contract_id = fuel_core_types::fuel_types::ContractId::try_from( + contract_created.contract_id.as_slice(), + ) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + let state_root = Bytes32::try_from(contract_created.state_root.as_slice()) + .map_err(|e| { + Error::Serialization(anyhow!( + "Could not convert state_root to Bytes32: {}", + e + )) + })?; + Ok(Output::contract_created(contract_id, state_root)) + } + } +} + +fn upgrade_purpose_from_proto( + proto: &ProtoUpgradePurpose, +) -> crate::result::Result { + match proto + .variant + .as_ref() + .ok_or_else(|| Error::Serialization(anyhow!("Missing upgrade purpose variant")))? + { + ProtoUpgradePurposeVariant::ConsensusParameters(consensus) => { + let witness_index = u16::try_from(consensus.witness_index).map_err(|e| { + Error::Serialization(anyhow!( + "Could not convert witness_index to u16: {}", + e + )) + })?; + let checksum = + Bytes32::try_from(consensus.checksum.as_slice()).map_err(|e| { + Error::Serialization(anyhow!( + "Could not convert checksum to Bytes32: {}", + e + )) + })?; + Ok(UpgradePurpose::ConsensusParameters { + witness_index, + checksum, + }) + } + ProtoUpgradePurposeVariant::StateTransition(state) => { + let root = Bytes32::try_from(state.root.as_slice()).map_err(|e| { + Error::Serialization(anyhow!( + "Could not convert state transition root to Bytes32: {}", + e + )) + })?; + Ok(UpgradePurpose::StateTransition { root }) + } + } +} + +fn utxo_id_from_proto(proto_utxo: &ProtoUtxoId) -> crate::result::Result { + let tx_id = Bytes32::try_from(proto_utxo.tx_id.as_slice()).map_err(|e| { + Error::Serialization(anyhow!("Could not convert tx_id to Bytes32: {}", e)) + })?; + let output_index = u16::try_from(proto_utxo.output_index).map_err(|e| { + Error::Serialization(anyhow!("Could not convert output_index to u16: {}", e)) + })?; + Ok(UtxoId::new(tx_id, output_index)) +} + +pub fn bytes32_to_vec(bytes: &fuel_core_types::fuel_types::Bytes32) -> Vec { + bytes.as_ref().to_vec() +} + +pub fn fuel_block_from_protobuf( + proto_block: ProtoBlock, + msg_ids: &[fuel_core_types::fuel_tx::MessageId], + event_inbox_root: Bytes32, +) -> crate::result::Result { + let versioned_block = proto_block + .versioned_block + .ok_or_else(|| anyhow::anyhow!("Missing protobuf versioned_block")) + .map_err(Error::Serialization)?; + let partial_header = match &versioned_block { + ProtoVersionedBlock::V1(v1_block) => { + let proto_header = v1_block + .header + .clone() + .ok_or_else(|| anyhow::anyhow!("Missing protobuf header")) + .map_err(Error::Serialization)?; + partial_header_from_proto_header(&proto_header)? + } + }; + let txs = match versioned_block { + ProtoVersionedBlock::V1(v1_inner) => v1_inner + .transactions + .iter() + .map(tx_from_proto_tx) + .collect::>()?, + }; + FuelBlock::new( + partial_header, + txs, + msg_ids, + event_inbox_root, + #[cfg(feature = "fault-proving")] + &ChainId::default(), + ) + .map_err(|e| anyhow!(e)) + .map_err(Error::Serialization) +} + +pub fn partial_header_from_proto_header( + proto_header: &ProtoHeader, +) -> crate::result::Result { + let partial_header = PartialBlockHeader { + consensus: proto_header_to_empty_consensus_header(&proto_header)?, + application: proto_header_to_empty_application_header(&proto_header)?, + }; + Ok(partial_header) +} + +pub fn tx_from_proto_tx( + proto_tx: &ProtoTransaction, +) -> crate::result::Result { + let variant = proto_tx + .variant + .as_ref() + .ok_or_else(|| Error::Serialization(anyhow!("Missing transaction variant")))?; + + match variant { + ProtoTransactionVariant::Script(proto_script) => { + let policies = proto_script + .policies + .clone() + .map(|p| policies_from_proto_policies(&p)) + .unwrap_or_default(); + let inputs = proto_script + .inputs + .iter() + .map(input_from_proto_input) + .collect::>>()?; + let outputs = proto_script + .outputs + .iter() + .map(output_from_proto_output) + .collect::>>()?; + let witnesses = proto_script + .witnesses + .iter() + .map(|w| Ok(Witness::from(w.clone()))) + .collect::>>()?; + let mut script_tx = FuelTransaction::script( + proto_script.script_gas_limit, + proto_script.script.clone(), + proto_script.script_data.clone(), + policies, + inputs, + outputs, + witnesses, + ); + *script_tx.receipts_root_mut() = Bytes32::try_from( + proto_script.receipts_root.as_slice(), + ) + .map_err(|e| { + Error::Serialization(anyhow!( + "Could not convert receipts_root to Bytes32: {}", + e + )) + })?; + + Ok(FuelTransaction::Script(script_tx)) + } + ProtoTransactionVariant::Create(proto_create) => { + let policies = proto_create + .policies + .clone() + .map(|p| policies_from_proto_policies(&p)) + .unwrap_or_default(); + let inputs = proto_create + .inputs + .iter() + .map(input_from_proto_input) + .collect::>>()?; + let outputs = proto_create + .outputs + .iter() + .map(output_from_proto_output) + .collect::>>()?; + let witnesses = proto_create + .witnesses + .iter() + .map(|w| Ok(Witness::from(w.clone()))) + .collect::>>()?; + let storage_slots = proto_create + .storage_slots + .iter() + .map(storage_slot_from_proto) + .collect::>>()?; + let salt = + fuel_core_types::fuel_types::Salt::try_from(proto_create.salt.as_slice()) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + let bytecode_witness_index = + u16::try_from(proto_create.bytecode_witness_index).map_err(|e| { + Error::Serialization(anyhow!( + "Could not convert bytecode_witness_index to u16: {}", + e + )) + })?; + + let create_tx = FuelTransaction::create( + bytecode_witness_index, + policies, + salt, + storage_slots, + inputs, + outputs, + witnesses, + ); + + Ok(FuelTransaction::Create(create_tx)) + } + ProtoTransactionVariant::Mint(proto_mint) => { + let tx_pointer_proto = proto_mint.tx_pointer.as_ref().ok_or_else(|| { + Error::Serialization(anyhow!("Missing tx_pointer on mint transaction")) + })?; + let tx_pointer = tx_pointer_from_proto(tx_pointer_proto)?; + let input_contract_proto = + proto_mint.input_contract.as_ref().ok_or_else(|| { + Error::Serialization(anyhow!( + "Missing input_contract on mint transaction" + )) + })?; + let input_contract = contract_input_from_proto(input_contract_proto)?; + let output_contract_proto = + proto_mint.output_contract.as_ref().ok_or_else(|| { + Error::Serialization(anyhow!( + "Missing output_contract on mint transaction" + )) + })?; + let output_contract = contract_output_from_proto(output_contract_proto)?; + let mint_asset_id = fuel_core_types::fuel_types::AssetId::try_from( + proto_mint.mint_asset_id.as_slice(), + ) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + + let mint_tx = FuelTransaction::mint( + tx_pointer, + input_contract, + output_contract, + proto_mint.mint_amount, + mint_asset_id, + proto_mint.gas_price, + ); + + Ok(FuelTransaction::Mint(mint_tx)) + } + ProtoTransactionVariant::Upgrade(proto_upgrade) => { + let purpose_proto = proto_upgrade.purpose.as_ref().ok_or_else(|| { + Error::Serialization(anyhow!("Missing purpose on upgrade transaction")) + })?; + let upgrade_purpose = upgrade_purpose_from_proto(purpose_proto)?; + let policies = proto_upgrade + .policies + .clone() + .map(|p| policies_from_proto_policies(&p)) + .unwrap_or_default(); + let inputs = proto_upgrade + .inputs + .iter() + .map(input_from_proto_input) + .collect::>>()?; + let outputs = proto_upgrade + .outputs + .iter() + .map(output_from_proto_output) + .collect::>>()?; + let witnesses = proto_upgrade + .witnesses + .iter() + .map(|w| Ok(Witness::from(w.clone()))) + .collect::>>()?; + + let upgrade_tx = FuelTransaction::upgrade( + upgrade_purpose, + policies, + inputs, + outputs, + witnesses, + ); + + Ok(FuelTransaction::Upgrade(upgrade_tx)) + } + ProtoTransactionVariant::Upload(proto_upload) => { + let policies = proto_upload + .policies + .clone() + .map(|p| policies_from_proto_policies(&p)) + .unwrap_or_default(); + let inputs = proto_upload + .inputs + .iter() + .map(input_from_proto_input) + .collect::>>()?; + let outputs = proto_upload + .outputs + .iter() + .map(output_from_proto_output) + .collect::>>()?; + let witnesses = proto_upload + .witnesses + .iter() + .map(|w| Ok(Witness::from(w.clone()))) + .collect::>>()?; + let root = Bytes32::try_from(proto_upload.root.as_slice()).map_err(|e| { + Error::Serialization(anyhow!( + "Could not convert upload root to Bytes32: {}", + e + )) + })?; + let witness_index = + u16::try_from(proto_upload.witness_index).map_err(|e| { + Error::Serialization(anyhow!( + "Could not convert witness_index to u16: {}", + e + )) + })?; + let subsection_index = + u16::try_from(proto_upload.subsection_index).map_err(|e| { + Error::Serialization(anyhow!( + "Could not convert subsection_index to u16: {}", + e + )) + })?; + let subsections_number = u16::try_from(proto_upload.subsections_number) + .map_err(|e| { + Error::Serialization(anyhow!( + "Could not convert subsections_number to u16: {}", + e + )) + })?; + let proof_set = proto_upload + .proof_set + .iter() + .map(|entry| { + Bytes32::try_from(entry.as_slice()).map_err(|e| { + Error::Serialization(anyhow!( + "Could not convert proof_set entry to Bytes32: {}", + e + )) + }) + }) + .collect::>>()?; + + let body = UploadBody { + root, + witness_index, + subsection_index, + subsections_number, + proof_set, + }; + + let upload_tx = + FuelTransaction::upload(body, policies, inputs, outputs, witnesses); + + Ok(FuelTransaction::Upload(upload_tx)) + } + ProtoTransactionVariant::Blob(proto_blob) => { + let policies = proto_blob + .policies + .clone() + .map(|p| policies_from_proto_policies(&p)) + .unwrap_or_default(); + let inputs = proto_blob + .inputs + .iter() + .map(input_from_proto_input) + .collect::>>()?; + let outputs = proto_blob + .outputs + .iter() + .map(output_from_proto_output) + .collect::>>()?; + let witnesses = proto_blob + .witnesses + .iter() + .map(|w| Ok(Witness::from(w.clone()))) + .collect::>>()?; + let blob_id = fuel_core_types::fuel_types::BlobId::try_from( + proto_blob.blob_id.as_slice(), + ) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + let witness_index = u16::try_from(proto_blob.witness_index).map_err(|e| { + Error::Serialization(anyhow!( + "Could not convert blob witness_index to u16: {}", + e + )) + })?; + let body = BlobBody { + id: blob_id, + witness_index, + }; + + let blob_tx = + FuelTransaction::blob(body, policies, inputs, outputs, witnesses); + + Ok(FuelTransaction::Blob(blob_tx)) + } + } +} + +fn input_from_proto_input(proto_input: &ProtoInput) -> crate::result::Result { + let variant = proto_input + .variant + .as_ref() + .ok_or_else(|| Error::Serialization(anyhow!("Missing input variant")))?; + + match variant { + ProtoInputVariant::CoinSigned(proto_coin_signed) => { + let utxo_proto = proto_coin_signed + .utxo_id + .as_ref() + .ok_or_else(|| Error::Serialization(anyhow!("Missing utxo_id")))?; + let utxo_id = utxo_id_from_proto(utxo_proto)?; + let owner = + Address::try_from(proto_coin_signed.owner.as_slice()).map_err(|e| { + Error::Serialization(anyhow!( + "Could not convert owner to Address: {}", + e + )) + })?; + let asset_id = fuel_core_types::fuel_types::AssetId::try_from( + proto_coin_signed.asset_id.as_slice(), + ) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + let tx_pointer_proto = proto_coin_signed + .tx_pointer + .as_ref() + .ok_or_else(|| Error::Serialization(anyhow!("Missing tx_pointer")))?; + let tx_pointer = tx_pointer_from_proto(tx_pointer_proto)?; + let witness_index = + u16::try_from(proto_coin_signed.witness_index).map_err(|e| { + Error::Serialization(anyhow!( + "Could not convert witness_index to u16: {}", + e + )) + })?; + + Ok(Input::coin_signed( + utxo_id, + owner, + proto_coin_signed.amount, + asset_id, + tx_pointer, + witness_index, + )) + } + ProtoInputVariant::CoinPredicate(proto_coin_predicate) => { + let utxo_proto = proto_coin_predicate + .utxo_id + .as_ref() + .ok_or_else(|| Error::Serialization(anyhow!("Missing utxo_id")))?; + let utxo_id = utxo_id_from_proto(utxo_proto)?; + let owner = Address::try_from(proto_coin_predicate.owner.as_slice()) + .map_err(|e| { + Error::Serialization(anyhow!( + "Could not convert owner to Address: {}", + e + )) + })?; + let asset_id = fuel_core_types::fuel_types::AssetId::try_from( + proto_coin_predicate.asset_id.as_slice(), + ) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + let tx_pointer_proto = proto_coin_predicate + .tx_pointer + .as_ref() + .ok_or_else(|| Error::Serialization(anyhow!("Missing tx_pointer")))?; + let tx_pointer = tx_pointer_from_proto(tx_pointer_proto)?; + + Ok(Input::coin_predicate( + utxo_id, + owner, + proto_coin_predicate.amount, + asset_id, + tx_pointer, + proto_coin_predicate.predicate_gas_used, + proto_coin_predicate.predicate.clone(), + proto_coin_predicate.predicate_data.clone(), + )) + } + ProtoInputVariant::Contract(proto_contract) => { + let contract = contract_input_from_proto(proto_contract)?; + Ok(Input::Contract(contract)) + } + ProtoInputVariant::MessageCoinSigned(proto_message) => { + let sender = + Address::try_from(proto_message.sender.as_slice()).map_err(|e| { + Error::Serialization(anyhow!( + "Could not convert sender to Address: {}", + e + )) + })?; + let recipient = Address::try_from(proto_message.recipient.as_slice()) + .map_err(|e| { + Error::Serialization(anyhow!( + "Could not convert recipient to Address: {}", + e + )) + })?; + let nonce = fuel_core_types::fuel_types::Nonce::try_from( + proto_message.nonce.as_slice(), + ) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + let witness_index = + u16::try_from(proto_message.witness_index).map_err(|e| { + Error::Serialization(anyhow!( + "Could not convert witness_index to u16: {}", + e + )) + })?; + + Ok(Input::message_coin_signed( + sender, + recipient, + proto_message.amount, + nonce, + witness_index, + )) + } + ProtoInputVariant::MessageCoinPredicate(proto_message) => { + let sender = + Address::try_from(proto_message.sender.as_slice()).map_err(|e| { + Error::Serialization(anyhow!( + "Could not convert sender to Address: {}", + e + )) + })?; + let recipient = Address::try_from(proto_message.recipient.as_slice()) + .map_err(|e| { + Error::Serialization(anyhow!( + "Could not convert recipient to Address: {}", + e + )) + })?; + let nonce = fuel_core_types::fuel_types::Nonce::try_from( + proto_message.nonce.as_slice(), + ) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + + Ok(Input::message_coin_predicate( + sender, + recipient, + proto_message.amount, + nonce, + proto_message.predicate_gas_used, + proto_message.predicate.clone(), + proto_message.predicate_data.clone(), + )) + } + ProtoInputVariant::MessageDataSigned(proto_message) => { + let sender = + Address::try_from(proto_message.sender.as_slice()).map_err(|e| { + Error::Serialization(anyhow!( + "Could not convert sender to Address: {}", + e + )) + })?; + let recipient = Address::try_from(proto_message.recipient.as_slice()) + .map_err(|e| { + Error::Serialization(anyhow!( + "Could not convert recipient to Address: {}", + e + )) + })?; + let nonce = fuel_core_types::fuel_types::Nonce::try_from( + proto_message.nonce.as_slice(), + ) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + let witness_index = + u16::try_from(proto_message.witness_index).map_err(|e| { + Error::Serialization(anyhow!( + "Could not convert witness_index to u16: {}", + e + )) + })?; + + Ok(Input::message_data_signed( + sender, + recipient, + proto_message.amount, + nonce, + witness_index, + proto_message.data.clone(), + )) + } + ProtoInputVariant::MessageDataPredicate(proto_message) => { + let sender = + Address::try_from(proto_message.sender.as_slice()).map_err(|e| { + Error::Serialization(anyhow!( + "Could not convert sender to Address: {}", + e + )) + })?; + let recipient = Address::try_from(proto_message.recipient.as_slice()) + .map_err(|e| { + Error::Serialization(anyhow!( + "Could not convert recipient to Address: {}", + e + )) + })?; + let nonce = fuel_core_types::fuel_types::Nonce::try_from( + proto_message.nonce.as_slice(), + ) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + + Ok(Input::message_data_predicate( + sender, + recipient, + proto_message.amount, + nonce, + proto_message.predicate_gas_used, + proto_message.data.clone(), + proto_message.predicate.clone(), + proto_message.predicate_data.clone(), + )) + } + } +} + +fn policies_from_proto_policies(proto_policies: &ProtoPolicies) -> FuelPolicies { + let ProtoPolicies { bits, values } = proto_policies; + let mut policies = FuelPolicies::default(); + let bits = + PoliciesBits::from_bits(*bits).expect("Should be able to create from `u32`"); + if bits.contains(PoliciesBits::Tip) + && let Some(tip) = values.first() + { + policies.set(PolicyType::Tip, Some(*tip)); + } + if bits.contains(PoliciesBits::WitnessLimit) + && let Some(witness_limit) = values.get(1) + { + policies.set(PolicyType::WitnessLimit, Some(*witness_limit)); + } + if bits.contains(PoliciesBits::Maturity) + && let Some(maturity) = values.get(2) + { + policies.set(PolicyType::Maturity, Some(*maturity)); + } + if bits.contains(PoliciesBits::MaxFee) + && let Some(max_fee) = values.get(3) + { + policies.set(PolicyType::MaxFee, Some(*max_fee)); + } + if bits.contains(PoliciesBits::Expiration) + && let Some(expiration) = values.get(4) + { + policies.set(PolicyType::Expiration, Some(*expiration)); + } + if bits.contains(PoliciesBits::Owner) + && let Some(owner) = values.get(5) + { + policies.set(PolicyType::Owner, Some(*owner)); + } + policies +} + +pub fn proto_header_to_empty_application_header( + proto_header: &ProtoHeader, +) -> crate::result::Result> { + match proto_header.versioned_header.clone() { + Some(ProtoVersionedHeader::V1(header)) => { + let app_header = ApplicationHeader { + da_height: DaBlockHeight::from(header.da_height), + consensus_parameters_version: header.consensus_parameters_version, + state_transition_bytecode_version: header + .state_transition_bytecode_version, + generated: Empty {}, + }; + Ok(app_header) + } + Some(ProtoVersionedHeader::V2(header)) => { + if cfg!(feature = "fault-proving") { + let app_header = ApplicationHeader { + da_height: DaBlockHeight::from(header.da_height), + consensus_parameters_version: header.consensus_parameters_version, + state_transition_bytecode_version: header + .state_transition_bytecode_version, + generated: Empty {}, + }; + Ok(app_header) + } else { + Err(anyhow!("V2 headers require the 'fault-proving' feature")) + .map_err(Error::Serialization) + } + } + None => Err(anyhow!("Missing protobuf versioned_header")) + .map_err(Error::Serialization), + } +} + +/// Alias the consensus header into an empty one. +pub fn proto_header_to_empty_consensus_header( + proto_header: &ProtoHeader, +) -> crate::result::Result> { + match proto_header.versioned_header.clone() { + Some(ProtoVersionedHeader::V1(header)) => { + let consensus_header = ConsensusHeader { + prev_root: *Bytes32::from_bytes_ref_checked(&header.prev_root).ok_or( + Error::Serialization(anyhow!("Could create `Bytes32` from bytes")), + )?, + height: header.height.into(), + time: tai64::Tai64(header.time), + generated: Empty {}, + }; + Ok(consensus_header) + } + Some(ProtoVersionedHeader::V2(header)) => { + if cfg!(feature = "fault-proving") { + let consensus_header = ConsensusHeader { + prev_root: *Bytes32::from_bytes_ref_checked(&header.prev_root) + .ok_or(Error::Serialization(anyhow!( + "Could create `Bytes32` from bytes" + )))?, + height: header.height.into(), + time: tai64::Tai64(header.time), + generated: Empty {}, + }; + Ok(consensus_header) + } else { + Err(anyhow!("V2 headers require the 'fault-proving' feature")) + .map_err(Error::Serialization) + } + } + None => Err(anyhow!("Missing protobuf versioned_header")) + .map_err(Error::Serialization), + } +} From 27f5bf1dba71179db464792c87e2b847bf12dcf9 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Fri, 14 Nov 2025 09:36:02 -0700 Subject: [PATCH 11/30] remove build for proto types --- Cargo.lock | 2 - .../services/block_aggregator_api/Cargo.toml | 4 - crates/services/block_aggregator_api/build.rs | 7 - .../block_aggregator_api/proto/api.proto | 412 ------------------ 4 files changed, 425 deletions(-) delete mode 100644 crates/services/block_aggregator_api/build.rs delete mode 100644 crates/services/block_aggregator_api/proto/api.proto diff --git a/Cargo.lock b/Cargo.lock index fc7e2004e83..f27c8ead3d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4267,8 +4267,6 @@ dependencies = [ "tokio", "tokio-stream", "tonic 0.14.2", - "tonic-prost", - "tonic-prost-build", "tracing", "tracing-subscriber", ] diff --git a/crates/services/block_aggregator_api/Cargo.toml b/crates/services/block_aggregator_api/Cargo.toml index aa0677686bf..1fb29a358f9 100644 --- a/crates/services/block_aggregator_api/Cargo.toml +++ b/crates/services/block_aggregator_api/Cargo.toml @@ -8,7 +8,6 @@ homepage = { workspace = true } license = { workspace = true } repository = { workspace = true } rust-version = { workspace = true } -build = "build.rs" [features] fault-proving = ["fuel-core-types/fault-proving"] @@ -37,11 +36,8 @@ thiserror = { workspace = true } tokio = { workspace = true } tokio-stream = { workspace = true } tonic = { workspace = true } -tonic-prost = { workspace = true } tracing = { workspace = true } -[build-dependencies] -tonic-prost-build = { workspace = true } [dev-dependencies] aws-sdk-s3 = { version = "1.111.0", features = ["test-util"] } diff --git a/crates/services/block_aggregator_api/build.rs b/crates/services/block_aggregator_api/build.rs deleted file mode 100644 index 190a1538000..00000000000 --- a/crates/services/block_aggregator_api/build.rs +++ /dev/null @@ -1,7 +0,0 @@ -fn main() -> Result<(), Box> { - tonic_prost_build::configure() - .type_attribute(".", "#[derive(serde::Serialize,serde::Deserialize)]") - .type_attribute(".", "#[allow(clippy::large_enum_variant)]") - .compile_protos(&["proto/api.proto"], &["proto/"])?; - Ok(()) -} diff --git a/crates/services/block_aggregator_api/proto/api.proto b/crates/services/block_aggregator_api/proto/api.proto deleted file mode 100644 index d1ef647f3ed..00000000000 --- a/crates/services/block_aggregator_api/proto/api.proto +++ /dev/null @@ -1,412 +0,0 @@ -syntax = "proto3"; - -package blockaggregator; - -message BlockHeightRequest {} - -message BlockHeightResponse { - optional uint32 height = 1; -} - -message BlockRangeRequest { - uint32 start = 1; - uint32 end = 2; -} - -message RemoteBlockRangeResponse { - string region = 1; - string bucket = 2; - string key = 3; - string url = 4; -} - -message Block { - oneof versioned_block { - V1Block v1 = 1; - } -} - -message V1Block { - Header header = 1; - repeated Transaction transactions = 2; -} - -message Header { - oneof versioned_header { - V1Header v1 = 1; - V2Header v2 = 2; - } -} - -message V1Header { - uint64 da_height = 1; - uint32 consensus_parameters_version = 2; - uint32 state_transition_bytecode_version = 3; - uint32 transactions_count = 4; - uint32 message_receipt_count = 5; - bytes transactions_root = 6; - bytes message_outbox_root = 7; - bytes event_inbox_root = 8; - bytes prev_root = 9; - uint32 height = 10; - uint64 time = 11; - bytes application_hash = 12; - optional bytes block_id = 13; -} - -message V2Header { - uint64 da_height = 1; - uint32 consensus_parameters_version = 2; - uint32 state_transition_bytecode_version = 3; - uint32 transactions_count = 4; - uint32 message_receipt_count = 5; - bytes transactions_root = 6; - bytes message_outbox_root = 7; - bytes event_inbox_root = 8; - bytes tx_id_commitment = 9; - bytes prev_root = 10; - uint32 height = 11; - uint64 time = 12; - bytes application_hash = 13; - optional bytes block_id = 14; -} - -message Transaction { - oneof variant { - ScriptTransaction script = 1; - CreateTransaction create = 2; - MintTransaction mint = 3; - UpgradeTransaction upgrade = 4; - UploadTransaction upload = 5; - BlobTransaction blob = 6; - } -} - -message ScriptTransaction { - uint64 script_gas_limit = 1; - bytes receipts_root = 2; - bytes script = 3; - bytes script_data = 4; - Policies policies = 5; - repeated Input inputs = 6; - repeated Output outputs = 7; - repeated bytes witnesses = 8; - ScriptMetadata metadata = 9; -} - -message CreateTransaction { - uint32 bytecode_witness_index = 1; - bytes salt = 2; - repeated StorageSlot storage_slots = 3; - Policies policies = 4; - repeated Input inputs = 5; - repeated Output outputs = 6; - repeated bytes witnesses = 7; - CreateMetadata metadata = 8; -} - -message MintTransaction { - TxPointer tx_pointer = 1; - ContractInput input_contract = 2; - ContractOutput output_contract = 3; - uint64 mint_amount = 4; - bytes mint_asset_id = 5; - uint64 gas_price = 6; - MintMetadata metadata = 7; -} - -message UpgradeTransaction { - UpgradePurpose purpose = 1; - Policies policies = 2; - repeated Input inputs = 3; - repeated Output outputs = 4; - repeated bytes witnesses = 5; - UpgradeMetadata metadata = 6; -} - -message UploadTransaction { - bytes root = 1; - uint32 witness_index = 2; - uint32 subsection_index = 3; - uint32 subsections_number = 4; - repeated bytes proof_set = 5; - Policies policies = 6; - repeated Input inputs = 7; - repeated Output outputs = 8; - repeated bytes witnesses = 9; - UploadMetadata metadata = 10; -} - -message BlobTransaction { - bytes blob_id = 1; - uint32 witness_index = 2; - Policies policies = 3; - repeated Input inputs = 4; - repeated Output outputs = 5; - repeated bytes witnesses = 6; - BlobMetadata metadata = 7; -} - -message Policies { - uint32 bits = 1; - repeated uint64 values = 2; -} - -message Input { - oneof variant { - CoinSignedInput coin_signed = 1; - CoinPredicateInput coin_predicate = 2; - ContractInput contract = 3; - MessageCoinSignedInput message_coin_signed = 4; - MessageCoinPredicateInput message_coin_predicate = 5; - MessageDataSignedInput message_data_signed = 6; - MessageDataPredicateInput message_data_predicate = 7; - } -} - -message CoinSignedInput { - UtxoId utxo_id = 1; - bytes owner = 2; - uint64 amount = 3; - bytes asset_id = 4; - TxPointer tx_pointer = 5; - uint32 witness_index = 6; - uint64 predicate_gas_used = 7; - bytes predicate = 8; - bytes predicate_data = 9; -} - -message CoinPredicateInput { - UtxoId utxo_id = 1; - bytes owner = 2; - uint64 amount = 3; - bytes asset_id = 4; - TxPointer tx_pointer = 5; - uint32 witness_index = 6; - uint64 predicate_gas_used = 7; - bytes predicate = 8; - bytes predicate_data = 9; -} - -message ContractInput { - UtxoId utxo_id = 1; - bytes balance_root = 2; - bytes state_root = 3; - TxPointer tx_pointer = 4; - bytes contract_id = 5; -} - -message MessageCoinSignedInput { - bytes sender = 1; - bytes recipient = 2; - uint64 amount = 3; - bytes nonce = 4; - uint32 witness_index = 5; - uint64 predicate_gas_used = 6; - bytes data = 7; - bytes predicate = 8; - bytes predicate_data = 9; -} - -message MessageCoinPredicateInput { - bytes sender = 1; - bytes recipient = 2; - uint64 amount = 3; - bytes nonce = 4; - uint32 witness_index = 5; - uint64 predicate_gas_used = 6; - bytes data = 7; - bytes predicate = 8; - bytes predicate_data = 9; -} - -message MessageDataSignedInput { - bytes sender = 1; - bytes recipient = 2; - uint64 amount = 3; - bytes nonce = 4; - uint32 witness_index = 5; - uint64 predicate_gas_used = 6; - bytes data = 7; - bytes predicate = 8; - bytes predicate_data = 9; -} - -message MessageDataPredicateInput { - bytes sender = 1; - bytes recipient = 2; - uint64 amount = 3; - bytes nonce = 4; - uint32 witness_index = 5; - uint64 predicate_gas_used = 6; - bytes data = 7; - bytes predicate = 8; - bytes predicate_data = 9; -} - -message Output { - oneof variant { - CoinOutput coin = 1; - ContractOutput contract = 2; - ChangeOutput change = 3; - VariableOutput variable = 4; - ContractCreatedOutput contract_created = 5; - } -} -message CoinOutput { - bytes to = 1; - uint64 amount = 2; - bytes asset_id = 3; -} -message ContractOutput { - uint32 input_index = 1; - bytes balance_root = 2; - bytes state_root = 3; -} -message ChangeOutput { - bytes to = 1; - uint64 amount = 2; - bytes asset_id = 3; -} -message VariableOutput { - bytes to = 1; - uint64 amount = 2; - bytes asset_id = 3; -} -message ContractCreatedOutput { - bytes contract_id = 1; - bytes state_root = 2; -} - -message UtxoId { - bytes tx_id = 1; - uint32 output_index = 2; -} - -message TxPointer { - uint32 block_height = 1; - uint32 tx_index = 2; -} - -message StorageSlot { - bytes key = 1; - bytes value = 2; -} - -message ScriptMetadata { - bytes id = 1; - uint32 inputs_offset = 2; - repeated uint32 inputs_offset_at = 3; - repeated PredicateOffset inputs_predicate_offset_at = 4; - uint32 outputs_offset = 5; - repeated uint32 outputs_offset_at = 6; - uint32 witnesses_offset = 7; - repeated uint32 witnesses_offset_at = 8; - uint64 script_gas_limit = 9; - bytes receipts_root = 10; - bytes script = 11; - bytes script_data = 12; -} - -message CreateMetadata { - bytes id = 1; - uint32 inputs_offset = 2; - repeated uint32 inputs_offset_at = 3; - repeated PredicateOffset inputs_predicate_offset_at = 4; - uint32 outputs_offset = 5; - repeated uint32 outputs_offset_at = 6; - uint32 witnesses_offset = 7; - repeated uint32 witnesses_offset_at = 8; - bytes contract_id = 9; - bytes contract_root = 10; - bytes state_root = 11; -} - -message MintMetadata { - bytes id = 1; -} - -message UpgradePurpose { - oneof variant { - UpgradeConsensusParameters consensus_parameters = 1; - UpgradeStateTransition state_transition = 2; - } -} - -message UpgradeConsensusParameters { - uint32 witness_index = 1; - bytes checksum = 2; -} - -message UpgradeStateTransition { - bytes root = 1; -} - -message UpgradeMetadata { - bytes id = 1; - uint32 inputs_offset = 2; - repeated uint32 inputs_offset_at = 3; - repeated PredicateOffset inputs_predicate_offset_at = 4; - uint32 outputs_offset = 5; - repeated uint32 outputs_offset_at = 6; - uint32 witnesses_offset = 7; - repeated uint32 witnesses_offset_at = 8; - oneof variant { - UpgradeConsensusParametersMetadata consensus_parameters = 9; - UpgradeStateTransitionMetadata state_transition = 10; - } -} - -message UpgradeConsensusParametersMetadata { - bytes consensus_parameters = 1; - bytes calculated_checksum = 2; -} - -message UpgradeStateTransitionMetadata {} - -message UploadMetadata { - bytes id = 1; - uint32 inputs_offset = 2; - repeated uint32 inputs_offset_at = 3; - repeated PredicateOffset inputs_predicate_offset_at = 4; - uint32 outputs_offset = 5; - repeated uint32 outputs_offset_at = 6; - uint32 witnesses_offset = 7; - repeated uint32 witnesses_offset_at = 8; -} - -message BlobMetadata { - bytes id = 1; - uint32 inputs_offset = 2; - repeated uint32 inputs_offset_at = 3; - repeated PredicateOffset inputs_predicate_offset_at = 4; - uint32 outputs_offset = 5; - repeated uint32 outputs_offset_at = 6; - uint32 witnesses_offset = 7; - repeated uint32 witnesses_offset_at = 8; -} - -message PredicateOffset { - optional InnerPredicateOffset offset = 1; -} - -message InnerPredicateOffset { - uint32 offset = 1; - uint32 length = 2; -} - - -message BlockResponse { - oneof payload { - Block literal = 1; - RemoteBlockRangeResponse remote = 2; - } -} - -message NewBlockSubscriptionRequest {} - -service BlockAggregator { - rpc GetBlockHeight (BlockHeightRequest) returns (BlockHeightResponse); - rpc GetBlockRange (BlockRangeRequest) returns (stream BlockResponse); - rpc NewBlockSubscription (NewBlockSubscriptionRequest) returns (stream BlockResponse); -} From 8bb2b15091353c023b1efb53934c65b77818b963 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Fri, 14 Nov 2025 10:05:00 -0700 Subject: [PATCH 12/30] Cleanup block creation in tests --- crates/types/src/blockchain/header.rs | 11 ---- crates/types/src/test_helpers.rs | 84 +++++++++++++-------------- 2 files changed, 42 insertions(+), 53 deletions(-) diff --git a/crates/types/src/blockchain/header.rs b/crates/types/src/blockchain/header.rs index 896cab86c3a..6620ee27cdf 100644 --- a/crates/types/src/blockchain/header.rs +++ b/crates/types/src/blockchain/header.rs @@ -79,17 +79,6 @@ impl BlockHeader { } } - /// Get the application portion of the header. - pub fn application_v1( - &self, - ) -> Option<&ApplicationHeader> { - match self { - BlockHeader::V1(header) => Some(header.application()), - #[cfg(feature = "fault-proving")] - BlockHeader::V2(_header) => None, - } - } - /// Get the consensus portion of the header. pub fn consensus(&self) -> &ConsensusHeader { match self { diff --git a/crates/types/src/test_helpers.rs b/crates/types/src/test_helpers.rs index 8c64ae90a10..0ecada057a7 100644 --- a/crates/types/src/test_helpers.rs +++ b/crates/types/src/test_helpers.rs @@ -1,11 +1,16 @@ +#[cfg(feature = "fault-proving")] +use crate::fuel_types::ChainId; use crate::{ blockchain::{ block::Block, header::{ - GeneratedConsensusFields, + ApplicationHeader, + BlockHeader, + BlockHeaderV1, + PartialBlockHeader, generate_txns_root, + v1::GeneratedApplicationFieldsV1, }, - primitives::DaBlockHeight, }, fuel_merkle::binary::root_calculator::MerkleRootCalculator, fuel_tx::{ @@ -46,7 +51,6 @@ use crate::{ }; use proptest::prelude::*; use rand::Rng; -use tai64::Tai64; /// Helper function to create a contract creation transaction /// from a given contract bytecode. @@ -612,41 +616,21 @@ fn arb_blob_transaction() -> impl Strategy { }) } -prop_compose! { - fn arb_consensus_header()( - prev_root in any::<[u8; 32]>(), - time in any::(), - ) -> crate::blockchain::header::ConsensusHeader { - crate::blockchain::header::ConsensusHeader { - prev_root: prev_root.into(), - height: BlockHeight::new(0), - time: Tai64(time), - generated: GeneratedConsensusFields::default(), - } - } -} - prop_compose! { /// Generate an arbitrary block with a variable number of transactions pub fn arb_block()( txs in arb_txs(), da_height in any::(), - consensus_parameter_version in any::(), + consensus_parameters_version in any::(), state_transition_bytecode_version in any::(), msg_ids in arb_msg_ids(), event_root in any::<[u8; 32]>(), - mut consensus_header in arb_consensus_header(), + chain_id in any::(), ) -> (Block, Vec, Bytes32) { - let mut fuel_block = Block::default(); - - *fuel_block.transactions_mut() = txs; - - fuel_block.header_mut().set_da_height(DaBlockHeight(da_height)); - fuel_block.header_mut().set_consensus_parameters_version(consensus_parameter_version); - fuel_block.header_mut().set_state_transition_bytecode_version(state_transition_bytecode_version); - - let count = fuel_block.transactions().len().try_into().expect("we shouldn't have more than u16::MAX transactions"); - let msg_root = msg_ids + let transactions_count = txs.len().try_into().expect("we shouldn't have more than u16::MAX transactions"); + let message_receipt_count = msg_ids.len().try_into().expect("we shouldn't have more than u32::MAX messages"); + let transactions_root = generate_txns_root(&txs); + let message_outbox_root = msg_ids .iter() .fold(MerkleRootCalculator::new(), |mut tree, id| { tree.push(id.as_ref()); @@ -654,19 +638,35 @@ prop_compose! { }) .root() .into(); - let tx_root = generate_txns_root(fuel_block.transactions()); - let event_root = event_root.into(); - fuel_block.header_mut().set_transactions_count(count); - fuel_block.header_mut().set_message_receipt_count(msg_ids.len().try_into().expect("we shouldn't have more than u32::MAX messages")); - fuel_block.header_mut().set_transaction_root(tx_root); - fuel_block.header_mut().set_message_outbox_root(msg_root); - fuel_block.header_mut().set_event_inbox_root(event_root); - - // Consensus - // TODO: Include V2 Application with V2 Header - let application_hash = fuel_block.header().application_v1().unwrap().hash(); - consensus_header.generated.application_hash = application_hash; - fuel_block.header_mut().set_consensus_header(consensus_header); + let event_root: Bytes32 = event_root.into(); + let header = { + let mut default = BlockHeaderV1::default(); + default.set_application_header(ApplicationHeader { + da_height: da_height.into(), + consensus_parameters_version, + state_transition_bytecode_version, + generated: GeneratedApplicationFieldsV1 { + transactions_count, + message_receipt_count, + transactions_root, + message_outbox_root, + event_inbox_root: event_root, + }, + }); + + BlockHeader::V1(default) + }; + let partial_block_header = PartialBlockHeader::from(&header); + #[cfg(feature = "fault-proving")] + let fuel_block = { + let chain_id = ChainId::new(chain_id); + Block::new(partial_block_header, txs, &msg_ids, event_root, &chain_id).unwrap() + }; + #[cfg(not(feature = "fault-proving"))] + let fuel_block = { + let _ = chain_id; + Block::new(partial_block_header, txs, &msg_ids, event_root).unwrap() + }; (fuel_block, msg_ids, event_root) } } From fd82db9b284c8a6559f35ed1106a7af2614d756b Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Fri, 14 Nov 2025 10:29:20 -0700 Subject: [PATCH 13/30] Fix stream in server response to be more streamlined (lol) --- .../src/api/protobuf_adapter.rs | 61 ++++++++----------- crates/types/src/test_helpers.rs | 4 +- 2 files changed, 27 insertions(+), 38 deletions(-) diff --git a/crates/services/block_aggregator_api/src/api/protobuf_adapter.rs b/crates/services/block_aggregator_api/src/api/protobuf_adapter.rs index 3e43e89a662..9eded8c5d4f 100644 --- a/crates/services/block_aggregator_api/src/api/protobuf_adapter.rs +++ b/crates/services/block_aggregator_api/src/api/protobuf_adapter.rs @@ -3,7 +3,10 @@ use crate::{ BlockAggregatorApi, BlockAggregatorQuery, }, - block_range_response::BlockRangeResponse, + block_range_response::{ + BlockRangeResponse, + BoxStream, + }, protobuf_types::{ Block as ProtoBlock, BlockHeightRequest as ProtoBlockHeightRequest, @@ -71,13 +74,13 @@ impl BlockAggregator for Server { ))), } } - type GetBlockRangeStream = ReceiverStream>; + // type GetBlockRangeStream = ReceiverStream>; + type GetBlockRangeStream = BoxStream>; async fn get_block_range( &self, request: tonic::Request, ) -> Result, tonic::Status> { - const ARB_LITERAL_BLOCK_BUFFER_SIZE: usize = 100; let req = request.into_inner(); let (response, receiver) = tokio::sync::oneshot::channel(); let query = BlockAggregatorQuery::GetBlockRange { @@ -93,50 +96,36 @@ impl BlockAggregator for Server { match res { Ok(block_range_response) => match block_range_response { BlockRangeResponse::Literal(inner) => { - let (tx, rx) = tokio::sync::mpsc::channel::< - Result, - >(ARB_LITERAL_BLOCK_BUFFER_SIZE); - - tokio::spawn(async move { - let mut s = inner; - while let Some(pb) = s.next().await { + let stream = inner + .map(|res| { let response = ProtoBlockResponse { - payload: Some(proto_block_response::Payload::Literal(pb)), + payload: Some(proto_block_response::Payload::Literal( + res, + )), }; - if tx.send(Ok(response)).await.is_err() { - break; - } - } - }); - - Ok(tonic::Response::new(ReceiverStream::new(rx))) + Ok(response) + }) + .boxed(); + Ok(tonic::Response::new(stream)) } BlockRangeResponse::Remote(inner) => { - let (tx, rx) = tokio::sync::mpsc::channel::< - Result, - >(ARB_LITERAL_BLOCK_BUFFER_SIZE); - - tokio::spawn(async move { - let mut s = inner; - while let Some(pb) = s.next().await { + let stream = inner + .map(|res| { let proto_response = ProtoRemoteBlockRangeResponse { - region: pb.region.clone(), - bucket: pb.bucket.clone(), - key: pb.key.clone(), - url: pb.url.clone(), + region: res.region.clone(), + bucket: res.bucket.clone(), + key: res.key.clone(), + url: res.url.clone(), }; let response = ProtoBlockResponse { payload: Some(proto_block_response::Payload::Remote( proto_response, )), }; - if tx.send(Ok(response)).await.is_err() { - break; - } - } - }); - - Ok(tonic::Response::new(ReceiverStream::new(rx))) + Ok(response) + }) + .boxed(); + Ok(tonic::Response::new(stream)) } }, Err(e) => Err(tonic::Status::internal(format!( diff --git a/crates/types/src/test_helpers.rs b/crates/types/src/test_helpers.rs index 0ecada057a7..1dbc750135d 100644 --- a/crates/types/src/test_helpers.rs +++ b/crates/types/src/test_helpers.rs @@ -488,7 +488,7 @@ fn arb_create_transaction() -> impl Strategy { ) .prop_map( |(policies, salt_bytes, storage_slots, inputs, outputs, witnesses)| { - let create = crate::fuel_tx::Transaction::create( + let create = Transaction::create( 0, policies, Salt::from(salt_bytes), @@ -520,7 +520,7 @@ fn arb_mint_transaction() -> impl Strategy { mint_asset_id, gas_price, )| { - let mint = crate::fuel_tx::Transaction::mint( + let mint = Transaction::mint( tx_pointer, input_contract, output_contract, From 56ece623fc9bebde50360738868a40254a7fd4b4 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Fri, 14 Nov 2025 10:32:25 -0700 Subject: [PATCH 14/30] Lint toml --- Cargo.toml | 2 +- crates/services/block_aggregator_api/Cargo.toml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 1142c760289..f03e1db5813 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -110,7 +110,7 @@ fuel-core-p2p = { version = "0.47.1", path = "./crates/services/p2p" } fuel-core-parallel-executor = { version = "0.47.1", path = "./crates/services/parallel-executor" } fuel-core-poa = { version = "0.47.1", path = "./crates/services/consensus_module/poa" } fuel-core-producer = { version = "0.47.1", path = "./crates/services/producer" } -fuel-core-protobuf = { version = "0.1.0", git = "https://github.com/FuelLabs/fuel-core-protobuf.git", branch = "add-defs" } +fuel-core-protobuf = { version = "0.1.0", git = "https://github.com/FuelLabs/fuel-core-protobuf.git", branch = "add-defs" } fuel-core-provider = { version = "0.47.1", path = "./crates/provider" } fuel-core-relayer = { version = "0.47.1", path = "./crates/services/relayer" } fuel-core-services = { version = "0.47.1", path = "./crates/services" } diff --git a/crates/services/block_aggregator_api/Cargo.toml b/crates/services/block_aggregator_api/Cargo.toml index 1fb29a358f9..0a94653bc63 100644 --- a/crates/services/block_aggregator_api/Cargo.toml +++ b/crates/services/block_aggregator_api/Cargo.toml @@ -38,7 +38,6 @@ tokio-stream = { workspace = true } tonic = { workspace = true } tracing = { workspace = true } - [dev-dependencies] aws-sdk-s3 = { version = "1.111.0", features = ["test-util"] } fuel-core-services = { workspace = true, features = ["test-helpers"] } From 7bcb4bd1e3be9c6a6d6b06367aac365b070189ff Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Fri, 14 Nov 2025 11:17:02 -0700 Subject: [PATCH 15/30] Move proto server to task --- .../src/api/protobuf_adapter.rs | 95 +++++++++++++------ .../src/api/protobuf_adapter/tests.rs | 8 +- .../services/block_aggregator_api/src/lib.rs | 10 +- 3 files changed, 74 insertions(+), 39 deletions(-) diff --git a/crates/services/block_aggregator_api/src/api/protobuf_adapter.rs b/crates/services/block_aggregator_api/src/api/protobuf_adapter.rs index 9eded8c5d4f..49d6d16539f 100644 --- a/crates/services/block_aggregator_api/src/api/protobuf_adapter.rs +++ b/crates/services/block_aggregator_api/src/api/protobuf_adapter.rs @@ -26,7 +26,16 @@ use crate::{ Result, }, }; +use anyhow::anyhow; use async_trait::async_trait; +use fuel_core_services::{ + RunnableService, + RunnableTask, + Service, + ServiceRunner, + StateWatcher, + TaskNextAction, +}; use futures::StreamExt; use tokio_stream::wrappers::ReceiverStream; use tonic::Status; @@ -167,42 +176,74 @@ impl BlockAggregator for Server { } pub struct ProtobufAPI { - _server_task_handle: tokio::task::JoinHandle<()>, - shutdown_sender: Option>, + _server_service: ServiceRunner, query_receiver: tokio::sync::mpsc::Receiver>, } -impl ProtobufAPI { - pub fn new(url: String) -> Self { - let (query_sender, query_receiver) = tokio::sync::mpsc::channel::< - BlockAggregatorQuery, - >(100); - let server = Server::new(query_sender); - let addr = url.parse().unwrap(); - let (shutdown_sender, shutdown_receiver) = tokio::sync::oneshot::channel::<()>(); - let _server_task_handle = tokio::spawn(async move { - let service = tonic::transport::Server::builder() - .add_service(ProtoBlockAggregatorServer::new(server)); - tokio::select! { - res = service.serve(addr) => { +pub struct ServerTask { + addr: std::net::SocketAddr, + query_sender: + tokio::sync::mpsc::Sender>, +} +#[async_trait::async_trait] +impl RunnableService for ServerTask { + const NAME: &'static str = "ProtobufServerTask"; + type SharedData = (); + type Task = Self; + type TaskParams = (); + + fn shared_data(&self) -> Self::SharedData {} + + async fn into_task( + self, + _state_watcher: &StateWatcher, + _params: Self::TaskParams, + ) -> anyhow::Result { + Ok(self) + } +} + +impl RunnableTask for ServerTask { + async fn run(&mut self, watcher: &mut StateWatcher) -> TaskNextAction { + let server = Server::new(self.query_sender.clone()); + let router = tonic::transport::Server::builder() + .add_service(ProtoBlockAggregatorServer::new(server)); + tokio::select! { + res = router.serve(self.addr) => { if let Err(e) = res { tracing::error!("BlockAggregator tonic server error: {}", e); + TaskNextAction::ErrorContinue(anyhow!(e)) } else { tracing::info!("BlockAggregator tonic server stopped"); + TaskNextAction::Stop } }, - _ = shutdown_receiver => { - tracing::info!("Shutting down BlockAggregator tonic server"); - }, + _ = watcher.while_started() => { + TaskNextAction::Stop } - }); - Self { - _server_task_handle, - shutdown_sender: Some(shutdown_sender), - query_receiver, } } + + async fn shutdown(self) -> anyhow::Result<()> { + Ok(()) + } +} + +impl ProtobufAPI { + pub fn new(url: String) -> Result { + let (query_sender, query_receiver) = tokio::sync::mpsc::channel::< + BlockAggregatorQuery, + >(100); + let addr = url.parse().unwrap(); + let _server_service = ServiceRunner::new(ServerTask { addr, query_sender }); + _server_service.start().map_err(|e| Error::Api(e))?; + let api = Self { + _server_service, + query_receiver, + }; + Ok(api) + } } impl BlockAggregatorApi for ProtobufAPI { @@ -220,11 +261,3 @@ impl BlockAggregatorApi for ProtobufAPI { Ok(query) } } - -impl Drop for ProtobufAPI { - fn drop(&mut self) { - if let Some(shutdown_sender) = self.shutdown_sender.take() { - let _ = shutdown_sender.send(()); - } - } -} diff --git a/crates/services/block_aggregator_api/src/api/protobuf_adapter/tests.rs b/crates/services/block_aggregator_api/src/api/protobuf_adapter/tests.rs index 95ca3045562..6940173fd27 100644 --- a/crates/services/block_aggregator_api/src/api/protobuf_adapter/tests.rs +++ b/crates/services/block_aggregator_api/src/api/protobuf_adapter/tests.rs @@ -45,7 +45,7 @@ fn free_local_addr() -> String { async fn await_query__get_current_height__client_receives_expected_value() { // given let path = free_local_addr(); - let mut api = ProtobufAPI::new(path.to_string()); + let mut api = ProtobufAPI::new(path.to_string()).unwrap(); tokio::time::sleep(std::time::Duration::from_millis(100)).await; // call get current height endpoint with client @@ -82,7 +82,7 @@ async fn await_query__get_current_height__client_receives_expected_value() { async fn await_query__get_block_range__client_receives_expected_value__literal() { // given let path = free_local_addr(); - let mut api = ProtobufAPI::new(path.to_string()); + let mut api = ProtobufAPI::new(path.to_string()).unwrap(); tokio::time::sleep(std::time::Duration::from_millis(100)).await; // call get current height endpoint with client @@ -156,7 +156,7 @@ async fn await_query__get_block_range__client_receives_expected_value__literal() async fn await_query__get_block_range__client_receives_expected_value__remote() { // given let path = free_local_addr(); - let mut api = ProtobufAPI::new(path.to_string()); + let mut api = ProtobufAPI::new(path.to_string()).unwrap(); tokio::time::sleep(std::time::Duration::from_millis(100)).await; // call get current height endpoint with client @@ -239,7 +239,7 @@ async fn await_query__get_block_range__client_receives_expected_value__remote() async fn await_query__new_block_stream__client_receives_expected_value() { // given let path = free_local_addr(); - let mut api = ProtobufAPI::new(path.to_string()); + let mut api = ProtobufAPI::new(path.to_string()).unwrap(); tokio::time::sleep(std::time::Duration::from_millis(100)).await; // call get current height endpoint with client diff --git a/crates/services/block_aggregator_api/src/lib.rs b/crates/services/block_aggregator_api/src/lib.rs index 8df0f0010ed..caa58b35c2a 100644 --- a/crates/services/block_aggregator_api/src/lib.rs +++ b/crates/services/block_aggregator_api/src/lib.rs @@ -71,14 +71,14 @@ pub mod integration { onchain_db: OnchainDB, importer: BoxStream, sync_from_height: BlockHeight, - ) -> ServiceRunner< + ) -> anyhow::Result, ProtoBlock, >, - > + >> where DB: BlockAggregatorDB< BlockRangeResponse = ::BlockRangeResponse, @@ -91,7 +91,8 @@ pub mod integration { E: std::fmt::Debug + Send + Sync, { let addr = config.addr.to_string(); - let api = ProtobufAPI::new(addr); + let api = ProtobufAPI::new(addr) + .map_err(|e| anyhow::anyhow!("Error creating API: {e}"))?; let db_ending_height = None; let block_source = ImporterAndDbSource::new( importer, @@ -106,7 +107,8 @@ pub mod integration { block_source, new_block_subscriptions: Vec::new(), }; - ServiceRunner::new(block_aggregator) + let runner = ServiceRunner::new(block_aggregator); + Ok(runner) } } From 6da16fdb72e33142f1bfa7f5ff0594bb3520882e Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Mon, 17 Nov 2025 07:48:05 -0700 Subject: [PATCH 16/30] Fix a bunch of feature compilation issues --- crates/fuel-core/src/combined_database.rs | 113 ++++++++++++------ crates/fuel-core/src/database.rs | 17 ++- .../src/database/database_description.rs | 1 + crates/fuel-core/src/service.rs | 2 + crates/fuel-core/src/service/sub_services.rs | 2 +- .../fuel_to_proto_conversions.rs | 2 + .../proto_to_fuel_conversions.rs | 2 + tests/tests/rpc.rs | 2 +- 8 files changed, 97 insertions(+), 44 deletions(-) diff --git a/crates/fuel-core/src/combined_database.rs b/crates/fuel-core/src/combined_database.rs index 3e1472cfa2c..2af72f8050a 100644 --- a/crates/fuel-core/src/combined_database.rs +++ b/crates/fuel-core/src/combined_database.rs @@ -3,6 +3,7 @@ use crate::state::{ historical_rocksdb::StateRewindPolicy, rocks_db::DatabaseConfig, }; +#[cfg(feature = "rpc")] use anyhow::anyhow; use crate::{ @@ -11,10 +12,6 @@ use crate::{ GenesisDatabase, Result as DatabaseResult, database_description::{ - block_aggregator::{ - BlockAggregatorDatabaseS3, - BlockAggregatorDatabaseStorage, - }, compression::CompressionDatabase, gas_price::GasPriceDatabase, off_chain::OffChain, @@ -24,6 +21,13 @@ use crate::{ }, service::DbType, }; + +#[cfg(feature = "rpc")] +use crate::database::database_description::block_aggregator::{ + BlockAggregatorDatabaseS3, + BlockAggregatorDatabaseStorage, +}; +#[cfg(feature = "rpc")] use fuel_core_block_aggregator_api::db::table::LatestBlock; #[cfg(feature = "test-helpers")] use fuel_core_chain_config::{ @@ -32,6 +36,7 @@ use fuel_core_chain_config::{ }; #[cfg(feature = "backup")] use fuel_core_services::TraceErr; +use fuel_core_storage::Result as StorageResult; #[cfg(feature = "test-helpers")] use fuel_core_storage::tables::{ Coins, @@ -41,9 +46,9 @@ use fuel_core_storage::tables::{ ContractsState, Messages, }; +#[cfg(feature = "rpc")] use fuel_core_storage::{ Error as StorageError, - Result as StorageResult, StorageAsRef, }; use fuel_core_types::{ @@ -70,7 +75,9 @@ pub struct CombinedDatabase { relayer: Database, gas_price: Database, compression: Database, + #[cfg(feature = "rpc")] block_aggregation_storage: Database, + #[cfg(feature = "rpc")] block_aggregation_s3: Database, } @@ -81,8 +88,10 @@ impl CombinedDatabase { relayer: Database, gas_price: Database, compression: Database, - block_aggregation_storage: Database, - block_aggregation_s3: Database, + #[cfg(feature = "rpc")] block_aggregation_storage: Database< + BlockAggregatorDatabaseStorage, + >, + #[cfg(feature = "rpc")] block_aggregation_s3: Database, ) -> Self { Self { on_chain, @@ -90,7 +99,9 @@ impl CombinedDatabase { relayer, gas_price, compression, + #[cfg(feature = "rpc")] block_aggregation_storage, + #[cfg(feature = "rpc")] block_aggregation_s3, } } @@ -102,7 +113,9 @@ impl CombinedDatabase { crate::state::rocks_db::RocksDb::::prune(path)?; crate::state::rocks_db::RocksDb::::prune(path)?; crate::state::rocks_db::RocksDb::::prune(path)?; + #[cfg(feature = "rpc")] crate::state::rocks_db::RocksDb::::prune(path)?; + #[cfg(feature = "rpc")] crate::state::rocks_db::RocksDb::::prune(path)?; Ok(()) } @@ -147,11 +160,13 @@ impl CombinedDatabase { crate::state::rocks_db::RocksDb::::backup(db_dir, temp_dir) .trace_err("Failed to backup compression database")?; + #[cfg(feature = "rpc")] crate::state::rocks_db::RocksDb::::backup( db_dir, temp_dir, ) .trace_err("Failed to backup block aggregation storage database")?; + #[cfg(feature = "rpc")] crate::state::rocks_db::RocksDb::::backup( db_dir, temp_dir, ) @@ -211,12 +226,14 @@ impl CombinedDatabase { ) .trace_err("Failed to restore compression database")?; + #[cfg(feature = "rpc")] crate::state::rocks_db::RocksDb::::restore( temp_restore_dir, backup_dir, ) .trace_err("Failed to restore block aggregation storage database")?; + #[cfg(feature = "rpc")] crate::state::rocks_db::RocksDb::::restore( temp_restore_dir, backup_dir, @@ -280,6 +297,7 @@ impl CombinedDatabase { ..database_config }, )?; + #[cfg(feature = "rpc")] let block_aggregation_storage = Database::open_rocksdb( path, state_rewind_policy, @@ -288,6 +306,7 @@ impl CombinedDatabase { ..database_config }, )?; + #[cfg(feature = "rpc")] let block_aggregation_s3 = Database::open_rocksdb( path, state_rewind_policy, @@ -303,7 +322,9 @@ impl CombinedDatabase { relayer, gas_price, compression, + #[cfg(feature = "rpc")] block_aggregation_storage, + #[cfg(feature = "rpc")] block_aggregation_s3, }) } @@ -320,7 +341,9 @@ impl CombinedDatabase { relayer: Default::default(), gas_price: Default::default(), compression: Default::default(), + #[cfg(feature = "rpc")] block_aggregation_storage: Default::default(), + #[cfg(feature = "rpc")] block_aggregation_s3: Default::default(), }) } @@ -367,7 +390,9 @@ impl CombinedDatabase { Database::in_memory(), Database::in_memory(), Database::in_memory(), + #[cfg(feature = "rpc")] Database::in_memory(), + #[cfg(feature = "rpc")] Database::in_memory(), ) } @@ -378,7 +403,9 @@ impl CombinedDatabase { self.relayer.check_version()?; self.gas_price.check_version()?; self.compression.check_version()?; + #[cfg(feature = "rpc")] self.block_aggregation_storage.check_version()?; + #[cfg(feature = "rpc")] self.block_aggregation_s3.check_version()?; Ok(()) } @@ -391,19 +418,23 @@ impl CombinedDatabase { &self.compression } + #[cfg(feature = "rpc")] pub fn block_aggregation_storage(&self) -> &Database { &self.block_aggregation_storage } + #[cfg(feature = "rpc")] pub fn block_aggregation_storage_mut( &mut self, ) -> &mut Database { &mut self.block_aggregation_storage } + #[cfg(feature = "rpc")] pub fn block_aggregation_s3(&self) -> &Database { &self.block_aggregation_s3 } + #[cfg(feature = "rpc")] pub fn block_aggregation_s3_mut( &mut self, ) -> &mut Database { @@ -516,27 +547,40 @@ impl CombinedDatabase { let compression_db_rolled_back = is_equal_or_none(compression_db_height, target_block_height); - let block_aggregation_storage_height = self - .block_aggregation_storage() - .storage_as_ref::() - .get(&()) - .map_err(|e: StorageError| anyhow!(e))? - .map(|b| b.into_owned()); - let block_aggregation_storage_rolled_back = is_equal_or_less_than_or_none( - block_aggregation_storage_height, - target_block_height, - ); - - let block_aggregation_s3_height = self - .block_aggregation_s3() - .storage_as_ref::() - .get(&()) - .map_err(|e: StorageError| anyhow!(e))? - .map(|b| b.into_owned()); - let block_aggregation_s3_rolled_back = is_equal_or_less_than_or_none( - block_aggregation_s3_height, - target_block_height, - ); + #[cfg(feature = "rpc")] + { + let block_aggregation_storage_height = self + .block_aggregation_storage() + .storage_as_ref::() + .get(&()) + .map_err(|e: StorageError| anyhow!(e))? + .map(|b| b.into_owned()); + let block_aggregation_storage_rolled_back = is_equal_or_less_than_or_none( + block_aggregation_storage_height, + target_block_height, + ); + + let block_aggregation_s3_height = self + .block_aggregation_s3() + .storage_as_ref::() + .get(&()) + .map_err(|e: StorageError| anyhow!(e))? + .map(|b| b.into_owned()); + let block_aggregation_s3_rolled_back = is_equal_or_less_than_or_none( + block_aggregation_s3_height, + target_block_height, + ); + + if !block_aggregation_storage_rolled_back { + self.block_aggregation_storage_mut() + .rollback_to(target_block_height)?; + } + + if !block_aggregation_s3_rolled_back { + self.block_aggregation_s3_mut() + .rollback_to(target_block_height)?; + } + } if on_chain_height == target_block_height && off_chain_height == target_block_height @@ -597,16 +641,6 @@ impl CombinedDatabase { { self.compression().rollback_last_block()?; } - - if !block_aggregation_storage_rolled_back { - self.block_aggregation_storage_mut() - .rollback_to(target_block_height)?; - } - - if !block_aggregation_s3_rolled_back { - self.block_aggregation_s3_mut() - .rollback_to(target_block_height)?; - } } if shutdown_listener.is_cancelled() { @@ -711,7 +745,9 @@ impl CombinedDatabase { self.relayer.shutdown(); self.gas_price.shutdown(); self.compression.shutdown(); + #[cfg(feature = "rpc")] self.block_aggregation_storage.shutdown(); + #[cfg(feature = "rpc")] self.block_aggregation_s3.shutdown(); } } @@ -744,6 +780,7 @@ fn is_equal_or_none(maybe_left: Option, right: T) -> bool { maybe_left.map(|left| left == right).unwrap_or(true) } +#[cfg(feature = "rpc")] fn is_equal_or_less_than_or_none(maybe_left: Option, right: T) -> bool { maybe_left.map(|left| left <= right).unwrap_or(true) } diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index bc712edf169..1f427ab3669 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -69,6 +69,11 @@ use std::{ pub type Result = core::result::Result; // TODO: Extract `Database` and all belongs into `fuel-core-database`. +#[cfg(feature = "rpc")] +use crate::database::database_description::block_aggregator::{ + BlockAggregatorDatabaseS3, + BlockAggregatorDatabaseStorage, +}; #[cfg(feature = "rocksdb")] use crate::state::{ historical_rocksdb::{ @@ -84,17 +89,17 @@ use crate::state::{ }; use crate::{ database::database_description::{ - block_aggregator::{ - BlockAggregatorDatabaseS3, - BlockAggregatorDatabaseStorage, - }, gas_price::GasPriceDatabase, indexation_availability, }, state::HeightType, }; + +#[cfg(feature = "rpc")] use anyhow::anyhow; +#[cfg(feature = "rpc")] use fuel_core_block_aggregator_api::db::table::LatestBlock; +#[cfg(feature = "rpc")] use fuel_core_storage::transactional::WriteTransaction; #[cfg(feature = "rocksdb")] use std::path::Path; @@ -448,6 +453,7 @@ impl Modifiable for Database { } } +#[cfg(feature = "rpc")] impl Modifiable for Database { fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { // Does not need to be monotonically increasing because @@ -456,6 +462,7 @@ impl Modifiable for Database { } } +#[cfg(feature = "rpc")] impl Database { pub fn rollback_to(&mut self, block_height: BlockHeight) -> StorageResult<()> { let mut tx = self.write_transaction(); @@ -467,6 +474,7 @@ impl Database { } } +#[cfg(feature = "rpc")] impl Modifiable for Database { fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { // Does not need to be monotonically increasing because @@ -475,6 +483,7 @@ impl Modifiable for Database { } } +#[cfg(feature = "rpc")] impl Database { pub fn rollback_to(&mut self, block_height: BlockHeight) -> StorageResult<()> { let mut tx = self.write_transaction(); diff --git a/crates/fuel-core/src/database/database_description.rs b/crates/fuel-core/src/database/database_description.rs index e991c2bc7f1..9a300158fd4 100644 --- a/crates/fuel-core/src/database/database_description.rs +++ b/crates/fuel-core/src/database/database_description.rs @@ -13,6 +13,7 @@ pub mod off_chain; pub mod on_chain; pub mod relayer; +#[cfg(feature = "rpc")] pub mod block_aggregator; pub trait DatabaseHeight: PartialEq + Default + Debug + Copy + Send + Sync { diff --git a/crates/fuel-core/src/service.rs b/crates/fuel-core/src/service.rs index 3b344f21559..a938da1dd94 100644 --- a/crates/fuel-core/src/service.rs +++ b/crates/fuel-core/src/service.rs @@ -193,7 +193,9 @@ impl FuelService { Default::default(), Default::default(), Default::default(), + #[cfg(feature = "rpc")] Default::default(), + #[cfg(feature = "rpc")] Default::default(), ); Self::from_combined_database(combined_database, config).await diff --git a/crates/fuel-core/src/service/sub_services.rs b/crates/fuel-core/src/service/sub_services.rs index 4cb189f0bdb..f3b057d9abc 100644 --- a/crates/fuel-core/src/service/sub_services.rs +++ b/crates/fuel-core/src/service/sub_services.rs @@ -529,7 +529,7 @@ pub fn init_sub_services( onchain_db, importer, sync_from_height, - ) + )? }; let graph_ql = fuel_core_graphql_api::api_service::new_service( diff --git a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/fuel_to_proto_conversions.rs b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/fuel_to_proto_conversions.rs index 4b22aaaab11..8eacfdf73e9 100644 --- a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/fuel_to_proto_conversions.rs +++ b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/fuel_to_proto_conversions.rs @@ -41,6 +41,8 @@ use crate::{ }, }; +#[cfg(feature = "fault-proving")] +use fuel_core_types::blockchain::header::BlockHeaderV2; use fuel_core_types::{ blockchain::{ header::{ diff --git a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/proto_to_fuel_conversions.rs b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/proto_to_fuel_conversions.rs index 6aec4246413..b4b073b1ce7 100644 --- a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/proto_to_fuel_conversions.rs +++ b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/proto_to_fuel_conversions.rs @@ -1,4 +1,6 @@ #[cfg(feature = "fault-proving")] +use crate::blocks::importer_and_db_source::serializer_adapter::ChainId; +#[cfg(feature = "fault-proving")] use crate::protobuf_types::V2Header as ProtoV2Header; use crate::{ protobuf_types::{ diff --git a/tests/tests/rpc.rs b/tests/tests/rpc.rs index df4263bf919..c0416784573 100644 --- a/tests/tests/rpc.rs +++ b/tests/tests/rpc.rs @@ -15,7 +15,7 @@ use fuel_core::{ }, }; use fuel_core_block_aggregator_api::{ - blocks::importer_and_db_source::serializer_adapter::fuel_block_from_protobuf, + blocks::importer_and_db_source::serializer_adapter::proto_to_fuel_conversions::fuel_block_from_protobuf, db::{ remote_cache::block_height_to_key, storage_or_remote_db::get_env_vars, From 7ec5dfb3c67c1d4887f12dcb1f1926dbed77d74f Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Mon, 17 Nov 2025 08:53:22 -0700 Subject: [PATCH 17/30] fix clippy warnings --- .github/workflows/ci.yml | 2 -- .../block_aggregator_api/src/api/protobuf_adapter.rs | 2 +- .../blocks/importer_and_db_source/serializer_adapter.rs | 9 ++------- .../serializer_adapter/fuel_to_proto_conversions.rs | 6 +++--- .../serializer_adapter/proto_to_fuel_conversions.rs | 6 ++---- 5 files changed, 8 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e2479d37a0a..24712adebb8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -174,8 +174,6 @@ jobs: uses: davidB/rust-cargo-make@v1 with: version: "0.36.4" - - name: Install Protoc - uses: arduino/setup-protoc@v3 - uses: rui314/setup-mold@v1 - uses: buildjet/cache@v3 with: diff --git a/crates/services/block_aggregator_api/src/api/protobuf_adapter.rs b/crates/services/block_aggregator_api/src/api/protobuf_adapter.rs index 49d6d16539f..d7c25937841 100644 --- a/crates/services/block_aggregator_api/src/api/protobuf_adapter.rs +++ b/crates/services/block_aggregator_api/src/api/protobuf_adapter.rs @@ -237,7 +237,7 @@ impl ProtobufAPI { >(100); let addr = url.parse().unwrap(); let _server_service = ServiceRunner::new(ServerTask { addr, query_sender }); - _server_service.start().map_err(|e| Error::Api(e))?; + _server_service.start().map_err(Error::Api)?; let api = Self { _server_service, query_receiver, diff --git a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter.rs b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter.rs index 57a08b0b30a..04247de5aa6 100644 --- a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter.rs +++ b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter.rs @@ -1,5 +1,3 @@ -#[cfg(feature = "fault-proving")] -use crate::protobuf_types::V2Header as ProtoV2Header; use crate::{ blocks::importer_and_db_source::BlockSerializer, protobuf_types::{ @@ -9,10 +7,7 @@ use crate::{ }, }; #[cfg(feature = "fault-proving")] -use fuel_core_types::{ - blockchain::header::BlockHeaderV2, - fuel_types::ChainId, -}; +use fuel_core_types::fuel_types::ChainId; use fuel_core_types::{ blockchain::{ @@ -35,7 +30,7 @@ impl BlockSerializer for SerializerAdapter { header: Some(proto_header), transactions: block .transactions() - .into_iter() + .iter() .map(proto_tx_from_tx) .collect(), }; diff --git a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/fuel_to_proto_conversions.rs b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/fuel_to_proto_conversions.rs index 8eacfdf73e9..8057c6494fe 100644 --- a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/fuel_to_proto_conversions.rs +++ b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/fuel_to_proto_conversions.rs @@ -96,7 +96,7 @@ pub fn proto_header_from_header(header: &BlockHeader) -> ProtoHeader { let versioned_header = match header { BlockHeader::V1(header) => { let proto_v1_header = - proto_v1_header_from_v1_header(&consensus, &block_id, header); + proto_v1_header_from_v1_header(consensus, &block_id, header); ProtoVersionedHeader::V1(proto_v1_header) } #[cfg(feature = "fault-proving")] @@ -471,7 +471,7 @@ fn proto_output_from_output(output: &Output) -> ProtoOutput { asset_id: asset_id.as_ref().to_vec(), }), Output::Contract(contract) => { - ProtoOutputVariant::Contract(proto_contract_output_from_contract(&contract)) + ProtoOutputVariant::Contract(proto_contract_output_from_contract(contract)) } Output::Change { to, @@ -496,7 +496,7 @@ fn proto_output_from_output(output: &Output) -> ProtoOutput { state_root, } => ProtoOutputVariant::ContractCreated(ProtoContractCreatedOutput { contract_id: contract_id.as_ref().to_vec(), - state_root: bytes32_to_vec(&state_root), + state_root: bytes32_to_vec(state_root), }), }; diff --git a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/proto_to_fuel_conversions.rs b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/proto_to_fuel_conversions.rs index b4b073b1ce7..5e69f063f72 100644 --- a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/proto_to_fuel_conversions.rs +++ b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/proto_to_fuel_conversions.rs @@ -1,7 +1,5 @@ #[cfg(feature = "fault-proving")] use crate::blocks::importer_and_db_source::serializer_adapter::ChainId; -#[cfg(feature = "fault-proving")] -use crate::protobuf_types::V2Header as ProtoV2Header; use crate::{ protobuf_types::{ Block as ProtoBlock, @@ -296,8 +294,8 @@ pub fn partial_header_from_proto_header( proto_header: &ProtoHeader, ) -> crate::result::Result { let partial_header = PartialBlockHeader { - consensus: proto_header_to_empty_consensus_header(&proto_header)?, - application: proto_header_to_empty_application_header(&proto_header)?, + consensus: proto_header_to_empty_consensus_header(proto_header)?, + application: proto_header_to_empty_application_header(proto_header)?, }; Ok(partial_header) } From 26c26e0abb424083443dc082e2e43da7fd2ce90c Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Mon, 17 Nov 2025 09:58:00 -0700 Subject: [PATCH 18/30] bump ci From 1da2683c90d5d27bb6078f1c10d4f442d130bce0 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Mon, 17 Nov 2025 10:06:42 -0700 Subject: [PATCH 19/30] update commit for proto types --- Cargo.lock | 128 ++++++++--------------------------------------------- 1 file changed, 18 insertions(+), 110 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f27c8ead3d9..dcd87ff602c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -423,7 +423,7 @@ dependencies = [ "alloy-rlp", "alloy-serde", "alloy-sol-types", - "itertools 0.14.0", + "itertools 0.13.0", "serde", "serde_json", "serde_with", @@ -3332,7 +3332,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d162beedaa69905488a8da94f5ac3edb4dd4788b732fadb7bd120b2625c1976" dependencies = [ "data-encoding", - "syn 2.0.107", + "syn 1.0.109", ] [[package]] @@ -3821,7 +3821,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4618,13 +4618,12 @@ dependencies = [ [[package]] name = "fuel-core-protobuf" version = "0.1.0" -source = "git+https://github.com/FuelLabs/fuel-core-protobuf.git?branch=add-defs#5b6e4d6c6db0ac654f152423cb0a6292fac9c7dc" +source = "git+https://github.com/FuelLabs/fuel-core-protobuf.git?branch=add-defs#74d5e499a7d22e1d9d97f409872dd9854b7d0629" dependencies = [ "prost 0.14.1", "serde", "tonic 0.14.2", "tonic-prost", - "tonic-prost-build", ] [[package]] @@ -5077,7 +5076,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cbdd607c9c70921cc016becde659e5062ae460b7bb3f525a1dd65f8209c0083" dependencies = [ "prost 0.12.6", - "prost-types 0.12.6", + "prost-types", "regex", "tonic 0.11.0", ] @@ -5952,7 +5951,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.1", + "socket2 0.5.10", "tokio", "tower-service", "tracing", @@ -7389,12 +7388,6 @@ dependencies = [ "unsigned-varint 0.8.0", ] -[[package]] -name = "multimap" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" - [[package]] name = "multistream-select" version = "0.13.0" @@ -7545,7 +7538,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7761,7 +7754,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -8425,28 +8418,6 @@ dependencies = [ "prost-derive 0.14.1", ] -[[package]] -name = "prost-build" -version = "0.14.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac6c3320f9abac597dcbc668774ef006702672474aad53c6d596b62e487b40b1" -dependencies = [ - "heck 0.5.0", - "itertools 0.14.0", - "log", - "multimap", - "once_cell", - "petgraph", - "prettyplease", - "prost 0.14.1", - "prost-types 0.14.1", - "pulldown-cmark", - "pulldown-cmark-to-cmark", - "regex", - "syn 2.0.107", - "tempfile", -] - [[package]] name = "prost-derive" version = "0.11.9" @@ -8480,7 +8451,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.107", @@ -8493,7 +8464,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.107", @@ -8508,15 +8479,6 @@ dependencies = [ "prost 0.12.6", ] -[[package]] -name = "prost-types" -version = "0.14.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9b4db3d6da204ed77bb26ba83b6122a73aeb2e87e25fbf7ad2e84c4ccbf8f72" -dependencies = [ - "prost 0.14.1", -] - [[package]] name = "psl-types" version = "2.0.11" @@ -8542,26 +8504,6 @@ dependencies = [ "psl-types", ] -[[package]] -name = "pulldown-cmark" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e8bbe1a966bd2f362681a44f6edce3c2310ac21e4d5067a6e7ec396297a6ea0" -dependencies = [ - "bitflags 2.9.4", - "memchr", - "unicase", -] - -[[package]] -name = "pulldown-cmark-to-cmark" -version = "21.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5b6a0769a491a08b31ea5c62494a8f144ee0987d86d670a8af4df1e1b7cde75" -dependencies = [ - "pulldown-cmark", -] - [[package]] name = "pulley-interpreter" version = "34.0.2" @@ -8683,7 +8625,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.1", "rustls 0.23.33", - "socket2 0.6.1", + "socket2 0.5.10", "thiserror 2.0.17", "tokio", "tracing", @@ -8720,9 +8662,9 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.1", + "socket2 0.5.10", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.52.0", ] [[package]] @@ -9293,7 +9235,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -10371,7 +10313,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -10389,7 +10331,7 @@ dependencies = [ "num-traits", "once_cell", "prost 0.12.6", - "prost-types 0.12.6", + "prost-types", "serde", "serde_bytes", "serde_json", @@ -10456,7 +10398,7 @@ dependencies = [ "bytes", "flex-error", "prost 0.12.6", - "prost-types 0.12.6", + "prost-types", "serde", "serde_bytes", "subtle-encoding", @@ -11047,18 +10989,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "tonic-build" -version = "0.14.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c40aaccc9f9eccf2cd82ebc111adc13030d23e887244bc9cfa5d1d636049de3" -dependencies = [ - "prettyplease", - "proc-macro2", - "quote", - "syn 2.0.107", -] - [[package]] name = "tonic-prost" version = "0.14.2" @@ -11070,22 +11000,6 @@ dependencies = [ "tonic 0.14.2", ] -[[package]] -name = "tonic-prost-build" -version = "0.14.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4a16cba4043dc3ff43fcb3f96b4c5c154c64cbd18ca8dce2ab2c6a451d058a2" -dependencies = [ - "prettyplease", - "proc-macro2", - "prost-build", - "prost-types 0.14.1", - "quote", - "syn 2.0.107", - "tempfile", - "tonic-build", -] - [[package]] name = "tower" version = "0.4.13" @@ -11339,12 +11253,6 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" -[[package]] -name = "unicase" -version = "2.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" - [[package]] name = "unicode-ident" version = "1.0.19" @@ -11881,7 +11789,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] From c1086084192b42ad126d6378701ec08d7d57bc26 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Mon, 17 Nov 2025 10:19:21 -0700 Subject: [PATCH 20/30] Fix compilation --- Cargo.lock | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dcd87ff602c..81f8ff4a064 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3821,7 +3821,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -4618,7 +4618,7 @@ dependencies = [ [[package]] name = "fuel-core-protobuf" version = "0.1.0" -source = "git+https://github.com/FuelLabs/fuel-core-protobuf.git?branch=add-defs#74d5e499a7d22e1d9d97f409872dd9854b7d0629" +source = "git+https://github.com/FuelLabs/fuel-core-protobuf.git?branch=add-defs#44c929564fcc92d0c3aaaab8099b2cb6283ecb72" dependencies = [ "prost 0.14.1", "serde", @@ -5951,7 +5951,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.1", "tokio", "tower-service", "tracing", @@ -8451,7 +8451,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.12.1", "proc-macro2", "quote", "syn 2.0.107", @@ -8464,7 +8464,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.12.1", "proc-macro2", "quote", "syn 2.0.107", @@ -8625,7 +8625,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.1", "rustls 0.23.33", - "socket2 0.5.10", + "socket2 0.6.1", "thiserror 2.0.17", "tokio", "tracing", @@ -8662,9 +8662,9 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.1", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -9235,7 +9235,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -10313,7 +10313,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] From 87d7b4a30b59092b86f03eb4e710a5dffcf3b635 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Mon, 17 Nov 2025 10:54:54 -0700 Subject: [PATCH 21/30] fix benches --- benches/benches/block_target_gas.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/benches/benches/block_target_gas.rs b/benches/benches/block_target_gas.rs index 792783333b4..d4dfa3a17c4 100644 --- a/benches/benches/block_target_gas.rs +++ b/benches/benches/block_target_gas.rs @@ -401,8 +401,6 @@ fn service_with_many_contracts( Default::default(), Default::default(), Default::default(), - Default::default(), - Default::default(), ), config.clone(), ) From 08ca30298b231b949b33342d48666c8cc4ee19db Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Mon, 17 Nov 2025 11:37:19 -0700 Subject: [PATCH 22/30] fix benches --- benches/Cargo.toml | 3 +++ benches/benches/block_target_gas.rs | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/benches/Cargo.toml b/benches/Cargo.toml index 4baceaa0968..1635956a9b5 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -16,6 +16,9 @@ fault-proving = [ "fuel-core-database/fault-proving", "fuel-core-sync/fault-proving", ] +rpc = [ + "fuel-core/rpc", +] [dependencies] anyhow = { workspace = true } diff --git a/benches/benches/block_target_gas.rs b/benches/benches/block_target_gas.rs index d4dfa3a17c4..80ad7697a83 100644 --- a/benches/benches/block_target_gas.rs +++ b/benches/benches/block_target_gas.rs @@ -401,6 +401,10 @@ fn service_with_many_contracts( Default::default(), Default::default(), Default::default(), + #[cfg(feature = "rpc")] + Default::default(), + #[cfg(feature = "rpc")] + Default::default(), ), config.clone(), ) From 71384d2b4c2b2a183f22427a936e73d2462736f1 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Mon, 17 Nov 2025 11:40:47 -0700 Subject: [PATCH 23/30] lint toml --- benches/Cargo.toml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/benches/Cargo.toml b/benches/Cargo.toml index 1635956a9b5..3d635e3fd6c 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -16,9 +16,7 @@ fault-proving = [ "fuel-core-database/fault-proving", "fuel-core-sync/fault-proving", ] -rpc = [ - "fuel-core/rpc", -] +rpc = ["fuel-core/rpc"] [dependencies] anyhow = { workspace = true } From 0cb9ec3a1aefaf7fd578be837b3b98ddbfab0f55 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Mon, 17 Nov 2025 15:14:55 -0700 Subject: [PATCH 24/30] use published version to get past ci checks --- Cargo.lock | 27 ++++++++++++++------------- Cargo.toml | 2 +- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9e267a14db1..c540df86e2c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -414,7 +414,7 @@ dependencies = [ "alloy-rlp", "alloy-serde", "alloy-sol-types", - "itertools 0.13.0", + "itertools 0.14.0", "serde", "serde_json", "serde_with", @@ -3324,7 +3324,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d162beedaa69905488a8da94f5ac3edb4dd4788b732fadb7bd120b2625c1976" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn 2.0.107", ] [[package]] @@ -3813,7 +3813,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4609,8 +4609,9 @@ dependencies = [ [[package]] name = "fuel-core-protobuf" -version = "0.1.0" -source = "git+https://github.com/FuelLabs/fuel-core-protobuf.git?branch=add-defs#44c929564fcc92d0c3aaaab8099b2cb6283ecb72" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3beed0ad34ae2f150f597a348e997a17b4991d32d002cd2de7c8e8f931bcc1b" dependencies = [ "prost 0.14.1", "serde", @@ -7524,7 +7525,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7731,7 +7732,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -8428,7 +8429,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.12.1", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.107", @@ -8441,7 +8442,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425" dependencies = [ "anyhow", - "itertools 0.12.1", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.107", @@ -8632,7 +8633,7 @@ dependencies = [ "once_cell", "socket2 0.6.1", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -9203,7 +9204,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -10281,7 +10282,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -11772,7 +11773,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index f03e1db5813..d012cba82ba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -110,7 +110,7 @@ fuel-core-p2p = { version = "0.47.1", path = "./crates/services/p2p" } fuel-core-parallel-executor = { version = "0.47.1", path = "./crates/services/parallel-executor" } fuel-core-poa = { version = "0.47.1", path = "./crates/services/consensus_module/poa" } fuel-core-producer = { version = "0.47.1", path = "./crates/services/producer" } -fuel-core-protobuf = { version = "0.1.0", git = "https://github.com/FuelLabs/fuel-core-protobuf.git", branch = "add-defs" } +fuel-core-protobuf = { version = "0.2.0" } fuel-core-provider = { version = "0.47.1", path = "./crates/provider" } fuel-core-relayer = { version = "0.47.1", path = "./crates/services/relayer" } fuel-core-services = { version = "0.47.1", path = "./crates/services" } From 5dd229b275b0e063f7bfb90b98c86a971f4445b6 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Mon, 17 Nov 2025 17:03:40 -0700 Subject: [PATCH 25/30] Fix compilation --- crates/fuel-core/Cargo.toml | 2 +- crates/fuel-core/src/service/sub_services.rs | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/crates/fuel-core/Cargo.toml b/crates/fuel-core/Cargo.toml index 237cf528451..d33caef2b64 100644 --- a/crates/fuel-core/Cargo.toml +++ b/crates/fuel-core/Cargo.toml @@ -60,7 +60,7 @@ clap = { workspace = true, features = ["derive"] } cosmrs = { version = "0.21", optional = true } derive_more = { version = "0.99" } enum-iterator = { workspace = true } -fuel-core-block-aggregator-api = { workspace = true, optional = true } +fuel-core-block-aggregator-api = { workspace = true } fuel-core-chain-config = { workspace = true, features = ["std"] } fuel-core-compression-service = { workspace = true } fuel-core-consensus-module = { workspace = true } diff --git a/crates/fuel-core/src/service/sub_services.rs b/crates/fuel-core/src/service/sub_services.rs index f3b057d9abc..12160c217d4 100644 --- a/crates/fuel-core/src/service/sub_services.rs +++ b/crates/fuel-core/src/service/sub_services.rs @@ -82,12 +82,15 @@ use fuel_core_gas_price_service::v1::{ uninitialized_task::new_gas_price_service_v1, }; use fuel_core_poa::Trigger; -#[cfg(feature = "rpc")] -use fuel_core_storage::StorageAsRef; use fuel_core_storage::{ self, transactional::AtomicView, }; +#[cfg(feature = "rpc")] +use fuel_core_storage::{ + Error as StorageError, + StorageAsRef, +}; #[cfg(feature = "relayer")] use fuel_core_types::blockchain::primitives::DaBlockHeight; @@ -483,7 +486,7 @@ pub fn init_sub_services( let maybe_sync_from_height = db .storage_as_ref::() .get(&()) - .map_err(|e| Error::DB(anyhow!(e)))? + .map_err(|e: StorageError| Error::DB(anyhow!(e)))? .map(|c| *c) .and_then(|h| h.succ()); sync_from_height = maybe_sync_from_height.unwrap_or(sync_from); @@ -513,7 +516,7 @@ pub fn init_sub_services( let maybe_sync_from_height = db .storage_as_ref::() .get(&()) - .map_err(|e| Error::DB(anyhow!(e)))? + .map_err(|e: StorageError| Error::DB(anyhow!(e)))? .map(|c| *c) .and_then(|h| h.succ()); sync_from_height = maybe_sync_from_height.unwrap_or(sync_from); From 779a13291b0c580910ee938781059d0441033f1a Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Mon, 17 Nov 2025 17:18:03 -0700 Subject: [PATCH 26/30] Add import --- bin/fuel-core/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/fuel-core/src/lib.rs b/bin/fuel-core/src/lib.rs index 704f0f3fa5b..67b5193783f 100644 --- a/bin/fuel-core/src/lib.rs +++ b/bin/fuel-core/src/lib.rs @@ -6,4 +6,5 @@ pub mod cli; pub use fuel_core::service::FuelService; +use fuel_core_block_aggregator_api as _; use tikv_jemallocator as _; // Used only by the binary From a4a7022bf8c60ceafa627c97a75a5ccace264d67 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Mon, 17 Nov 2025 17:24:27 -0700 Subject: [PATCH 27/30] Add correct import --- bin/fuel-core/src/lib.rs | 1 - crates/fuel-core/src/lib.rs | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/fuel-core/src/lib.rs b/bin/fuel-core/src/lib.rs index 67b5193783f..704f0f3fa5b 100644 --- a/bin/fuel-core/src/lib.rs +++ b/bin/fuel-core/src/lib.rs @@ -6,5 +6,4 @@ pub mod cli; pub use fuel_core::service::FuelService; -use fuel_core_block_aggregator_api as _; use tikv_jemallocator as _; // Used only by the binary diff --git a/crates/fuel-core/src/lib.rs b/crates/fuel-core/src/lib.rs index d464a46d073..ea490b8c1b2 100644 --- a/crates/fuel-core/src/lib.rs +++ b/crates/fuel-core/src/lib.rs @@ -55,6 +55,8 @@ pub mod state; // In the future this module will be a separate crate for `fuel-core-graphql-api`. mod graphql_api; +use fuel_core_block_aggregator_api as _; + pub mod fuel_core_graphql_api { pub use crate::graphql_api::*; } From 345304c9ac883b4880962519586ebbd0f9fc05c4 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 19 Nov 2025 11:41:17 -0700 Subject: [PATCH 28/30] fix sync --- .../src/blocks/importer_and_db_source.rs | 3 +- .../importer_and_db_source/sync_service.rs | 41 +++---- .../blocks/importer_and_db_source/tests.rs | 106 +----------------- .../services/block_aggregator_api/src/lib.rs | 7 +- 4 files changed, 25 insertions(+), 132 deletions(-) diff --git a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source.rs b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source.rs index 497aab2ec9b..513d8a5259a 100644 --- a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source.rs +++ b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source.rs @@ -73,7 +73,7 @@ where serializer: Serializer, database: DB, db_starting_height: BlockHeight, - db_ending_height: Option, + db_ending_height: BlockHeight, ) -> Self { const ARB_CHANNEL_SIZE: usize = 100; let (block_return, receiver) = tokio::sync::mpsc::channel(ARB_CHANNEL_SIZE); @@ -92,7 +92,6 @@ where database, db_starting_height, db_ending_height, - new_end_receiver, ); let sync_runner = ServiceRunner::new(sync_task); sync_runner.start().unwrap(); diff --git a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/sync_service.rs b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/sync_service.rs index be8b6b19e94..394b30a7f6c 100644 --- a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/sync_service.rs +++ b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/sync_service.rs @@ -33,8 +33,8 @@ pub struct SyncTask { block_return_sender: Sender>, db: DB, next_height: BlockHeight, - maybe_stop_height: Option, - new_ending_height: tokio::sync::oneshot::Receiver, + // exclusive, does not ask for this block + stop_height: BlockHeight, } impl SyncTask @@ -49,23 +49,15 @@ where block_return: Sender>, db: DB, db_starting_height: BlockHeight, - db_ending_height: Option, - new_ending_height: tokio::sync::oneshot::Receiver, + // does not ask for this block + db_ending_height: BlockHeight, ) -> Self { Self { serializer, block_return_sender: block_return, db, next_height: db_starting_height, - maybe_stop_height: db_ending_height, - new_ending_height, - } - } - - async fn maybe_update_stop_height(&mut self) { - if let Ok(last_height) = self.new_ending_height.try_recv() { - tracing::info!("updating last height to {}", last_height); - self.maybe_stop_height = Some(last_height); + stop_height: db_ending_height, } } @@ -99,12 +91,6 @@ where } Ok(txs) } - - // For now just have arbitrary 10 ms sleep to avoid busy looping. - // This could be more complicated with increasing backoff times, etc. - async fn go_to_sleep_before_continuing(&self) { - tokio::time::sleep(Duration::from_millis(10)).await; - } } impl RunnableTask for SyncTask @@ -117,13 +103,10 @@ where E: std::fmt::Debug + Send, { async fn run(&mut self, _watcher: &mut StateWatcher) -> TaskNextAction { - self.maybe_update_stop_height().await; - if let Some(last_height) = self.maybe_stop_height - && self.next_height >= last_height - { + if self.next_height >= self.stop_height { tracing::info!( - "reached end height {}, putting task into hibernation", - last_height + "reached stop height {}, putting task into hibernation", + self.stop_height ); futures::future::pending().await } @@ -133,6 +116,10 @@ where tracing::error!("error fetching block at height {}: {:?}", next_height, e); }); if let Some(block) = maybe_block { + tracing::debug!( + "found block at height {:?}, sending to return channel", + next_height + ); let res = self.serializer.serialize_block(&block); let block = try_or_continue!(res); let event = @@ -140,11 +127,11 @@ where let res = self.block_return_sender.send(event).await; try_or_continue!(res); self.next_height = BlockHeight::from((*next_height).saturating_add(1)); + TaskNextAction::Continue } else { tracing::warn!("no block found at height {:?}, retrying", next_height); - self.go_to_sleep_before_continuing().await; + TaskNextAction::Stop } - TaskNextAction::Continue } async fn shutdown(self) -> anyhow::Result<()> { diff --git a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/tests.rs b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/tests.rs index 3820fafbf0e..7c5c578d53f 100644 --- a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/tests.rs +++ b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/tests.rs @@ -72,12 +72,14 @@ async fn next_block__gets_new_block_from_importer() { let serializer = MockSerializer; let db = database(); let db_starting_height = BlockHeight::from(0u32); + // we don't need to sync anything, so we can use the same height for both + let db_ending_height = db_starting_height; let mut adapter = ImporterAndDbSource::new( block_stream, serializer.clone(), db, db_starting_height, - None, + db_ending_height, ); // when @@ -121,7 +123,7 @@ async fn next_block__can_get_block_from_db() { tx.commit().unwrap(); let block_stream = tokio_stream::pending().into_boxed(); let db_starting_height = *height; - let db_ending_height = Some(height2); + let db_ending_height = height2; let mut adapter = ImporterAndDbSource::new( block_stream, serializer.clone(), @@ -138,103 +140,3 @@ async fn next_block__can_get_block_from_db() { let expected = BlockSourceEvent::OldBlock(*height, serialized); assert_eq!(expected, actual); } - -#[tokio::test] -async fn next_block__will_sync_blocks_from_db_after_receiving_height_from_new_end() { - // given - let chain_id = ChainId::default(); - let height1 = BlockHeight::from(0u32); - let height2 = BlockHeight::from(1u32); - let height3 = BlockHeight::from(2u32); - let block1 = arbitrary_block_with_txs(height1); - let block2 = arbitrary_block_with_txs(height2); - let serializer = MockSerializer; - let mut db = database(); - let mut tx = db.write_transaction(); - let compressed_block = block1.compress(&chain_id); - tx.storage_as_mut::() - .insert(&height1, &compressed_block) - .unwrap(); - tx.storage_as_mut::() - .insert( - &block1.transactions()[0].id(&chain_id), - &block1.transactions()[0], - ) - .unwrap(); - tx.commit().unwrap(); - let mut tx = db.write_transaction(); - let compressed_block = block2.compress(&chain_id); - tx.storage_as_mut::() - .insert(&height2, &compressed_block) - .unwrap(); - tx.storage_as_mut::() - .insert( - &block2.transactions()[0].id(&chain_id), - &block2.transactions()[0], - ) - .unwrap(); - tx.commit().unwrap(); - - // Add the imported block to db as well as streaming - let block3 = arbitrary_block_with_txs(height3); - let mut tx = db.write_transaction(); - let compressed_block = block3.compress(&chain_id); - tx.storage_as_mut::() - .insert(&height3, &compressed_block) - .unwrap(); - tx.storage_as_mut::() - .insert( - &block3.transactions()[0].id(&chain_id), - &block3.transactions()[0], - ) - .unwrap(); - tx.commit().unwrap(); - - let sealed_block = SealedBlock { - entity: block3.clone(), - consensus: Default::default(), - }; - let import_result = Arc::new( - ImportResult { - sealed_block, - tx_status: vec![], - events: vec![], - source: Default::default(), - } - .wrap(), - ); - let blocks: Vec = vec![import_result]; - let block_stream = stream_with_pending(blocks); - let db_starting_height = height1; - let mut adapter = ImporterAndDbSource::new( - block_stream, - serializer.clone(), - db, - db_starting_height, - None, - ); - - // when - let actual1 = adapter.next_block().await.unwrap(); - let actual2 = adapter.next_block().await.unwrap(); - let actual3 = adapter.next_block().await.unwrap(); - - // then - let actual = vec![actual1, actual2, actual3] - .into_iter() - .collect::>(); - // should receive the - let expected = vec![ - BlockSourceEvent::OldBlock(height1, serializer.serialize_block(&block1).unwrap()), - BlockSourceEvent::OldBlock(height2, serializer.serialize_block(&block2).unwrap()), - BlockSourceEvent::NewBlock(height3, serializer.serialize_block(&block3).unwrap()), - ]; - let expected: HashSet<_> = expected.into_iter().collect(); - let length = actual.len(); - let expected_length = expected.len(); - for event in &actual { - tracing::debug!("actual event: {:?}", event); - } - assert_eq!(length, expected_length); - assert_eq!(expected, actual); -} diff --git a/crates/services/block_aggregator_api/src/lib.rs b/crates/services/block_aggregator_api/src/lib.rs index caa58b35c2a..90572e2cf24 100644 --- a/crates/services/block_aggregator_api/src/lib.rs +++ b/crates/services/block_aggregator_api/src/lib.rs @@ -50,6 +50,7 @@ pub mod integration { FuelBlocks, Transactions, }, + transactional::HistoricalView, }; use fuel_core_types::{ fuel_types::BlockHeight, @@ -88,12 +89,16 @@ pub mod integration { OnchainDB: Send + Sync, OnchainDB: StorageInspect, OnchainDB: StorageInspect, + OnchainDB: HistoricalView, E: std::fmt::Debug + Send + Sync, { let addr = config.addr.to_string(); let api = ProtobufAPI::new(addr) .map_err(|e| anyhow::anyhow!("Error creating API: {e}"))?; - let db_ending_height = None; + let db_ending_height = onchain_db + .latest_height() + .and_then(BlockHeight::succ) + .unwrap_or(BlockHeight::from(0)); let block_source = ImporterAndDbSource::new( importer, serializer, From 02403451169edac2f34053f21d633423a573093e Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 19 Nov 2025 11:58:31 -0700 Subject: [PATCH 29/30] Appease Clippy-sama, remove unused code in importer --- .../src/blocks/importer_and_db_source.rs | 9 ++----- .../importer_service.rs | 24 +------------------ .../importer_and_db_source/sync_service.rs | 1 - .../blocks/importer_and_db_source/tests.rs | 5 ---- 4 files changed, 3 insertions(+), 36 deletions(-) diff --git a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source.rs b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source.rs index 513d8a5259a..3bf49003ccc 100644 --- a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source.rs +++ b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source.rs @@ -77,13 +77,8 @@ where ) -> Self { const ARB_CHANNEL_SIZE: usize = 100; let (block_return, receiver) = tokio::sync::mpsc::channel(ARB_CHANNEL_SIZE); - let (new_end_sender, new_end_receiver) = tokio::sync::oneshot::channel(); - let importer_task = ImporterTask::new( - importer, - serializer.clone(), - block_return.clone(), - Some(new_end_sender), - ); + let importer_task = + ImporterTask::new(importer, serializer.clone(), block_return.clone()); let importer_runner = ServiceRunner::new(importer_task); importer_runner.start().unwrap(); let sync_task = SyncTask::new( diff --git a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/importer_service.rs b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/importer_service.rs index 74151e2a0c7..b939400811c 100644 --- a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/importer_service.rs +++ b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/importer_service.rs @@ -11,10 +11,7 @@ use fuel_core_services::{ try_or_continue, try_or_stop, }; -use fuel_core_types::{ - fuel_types::BlockHeight, - services::block_importer::SharedImportResult, -}; +use fuel_core_types::services::block_importer::SharedImportResult; use futures::StreamExt; use tokio::sync::mpsc::Sender; @@ -22,7 +19,6 @@ pub struct ImporterTask { importer: BoxStream, serializer: Serializer, block_return_sender: Sender>, - new_end_sender: Option>, } impl ImporterTask @@ -34,13 +30,11 @@ where importer: BoxStream, serializer: Serializer, block_return: Sender>, - new_end_sender: Option>, ) -> Self { Self { importer, serializer, block_return_sender: block_return, - new_end_sender, } } } @@ -75,22 +69,6 @@ where match maybe_import_result { Some(import_result) => { let height = import_result.sealed_block.entity.header().height(); - if let Some(sender) = self.new_end_sender.take() { - match sender.send(*height) { - Ok(_) => { - tracing::debug!( - "sent new end height to sync task: {:?}", - height - ); - } - Err(e) => { - tracing::error!( - "failed to send new end height to sync task: {:?}", - e - ); - } - } - } let res = self .serializer .serialize_block(&import_result.sealed_block.entity); diff --git a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/sync_service.rs b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/sync_service.rs index 394b30a7f6c..59791bf0d61 100644 --- a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/sync_service.rs +++ b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/sync_service.rs @@ -25,7 +25,6 @@ use fuel_core_types::{ }, fuel_types::BlockHeight, }; -use std::time::Duration; use tokio::sync::mpsc::Sender; pub struct SyncTask { diff --git a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/tests.rs b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/tests.rs index 7c5c578d53f..a04023ea537 100644 --- a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/tests.rs +++ b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/tests.rs @@ -18,7 +18,6 @@ use fuel_core_storage::{ }, }; use futures::StreamExt; -use std::collections::HashSet; use fuel_core_types::{ blockchain::SealedBlock, @@ -49,10 +48,6 @@ fn database() -> StorageTransaction> { InMemoryStorage::default().into_transaction() } -fn stream_with_pending(items: Vec) -> BoxStream { - tokio_stream::iter(items).chain(pending()).into_boxed() -} - #[tokio::test] async fn next_block__gets_new_block_from_importer() { // given From b6b97cd779cd47cb4ccdc57a3845c97b201a6d4c Mon Sep 17 00:00:00 2001 From: Mitchell Turner Date: Tue, 25 Nov 2025 08:15:30 -0700 Subject: [PATCH 30/30] Improve S3 Configuration (#3147) ## Linked Issues/PRs Update to use the protobuf definitions here: https://github.com/FuelLabs/fuel-core-protobuf/pull/2 ## Description ## Checklist - [ ] Breaking changes are clearly marked as such in the PR description and changelog - [ ] New behavior is reflected in tests - [ ] [The specification](https://github.com/FuelLabs/fuel-specs/) matches the implemented behavior (link update PR if changes are needed) ### Before requesting review - [ ] I have reviewed the code myself - [ ] I have created follow-up issues caused by this PR and linked them here ### After merging, notify other teams [Add or remove entries as needed] - [ ] [Rust SDK](https://github.com/FuelLabs/fuels-rs/) - [ ] [Sway compiler](https://github.com/FuelLabs/sway/) - [ ] [Platform documentation](https://github.com/FuelLabs/devrel-requests/issues/new?assignees=&labels=new+request&projects=&template=NEW-REQUEST.yml&title=%5BRequest%5D%3A+) (for out-of-organization contributors, the person merging the PR will do this) - [ ] Someone else? --- .../actions/slack-notify-template/action.yml | 25 ++ .github/workflows/ci.yml | 13 +- .github/workflows/docker-images.yml | 4 +- Cargo.lock | 40 ++- Cargo.toml | 8 +- bin/fuel-core/src/cli/run.rs | 4 + bin/fuel-core/src/cli/run/rpc.rs | 41 ++- bin/fuel-core/src/lib.rs | 1 + crates/fuel-core/Cargo.toml | 3 +- crates/fuel-core/src/query/message.rs | 2 +- crates/fuel-core/src/schema/tx/assemble_tx.rs | 16 +- crates/fuel-core/src/service.rs | 2 +- crates/fuel-core/src/service/adapters.rs | 3 + crates/fuel-core/src/service/adapters/rpc.rs | 53 ++++ .../src/service/adapters/rpc/tests.rs | 46 +++ crates/fuel-core/src/service/config.rs | 3 + crates/fuel-core/src/service/sub_services.rs | 89 +++--- .../services/block_aggregator_api/Cargo.toml | 2 + .../services/block_aggregator_api/src/api.rs | 5 +- .../src/api/protobuf_adapter.rs | 31 +- .../src/api/protobuf_adapter/tests.rs | 71 +++-- .../src/block_aggregator.rs | 4 +- .../src/block_range_response.rs | 13 +- .../src/blocks/importer_and_db_source.rs | 51 ++-- .../importer_service.rs | 8 +- .../serializer_adapter.rs | 27 +- .../fuel_to_proto_conversions.rs | 289 ++++++++++++++++++ .../proto_to_fuel_conversions.rs | 278 ++++++++++++++++- .../importer_and_db_source/sync_service.rs | 93 ++++-- .../blocks/importer_and_db_source/tests.rs | 77 +++-- .../src/db/remote_cache.rs | 94 ++++-- .../src/db/remote_cache/tests.rs | 82 ++--- .../block_aggregator_api/src/db/storage_db.rs | 4 +- .../src/db/storage_db/tests.rs | 6 +- .../src/db/storage_or_remote_db.rs | 57 +--- .../services/block_aggregator_api/src/lib.rs | 28 +- .../block_aggregator_api/src/result.rs | 10 + .../block_aggregator_api/src/tests.rs | 2 +- .../consensus_module/poa/src/service.rs | 4 +- crates/services/executor/src/executor.rs | 26 +- crates/services/p2p/src/peer_manager.rs | 2 +- .../services/producer/src/block_producer.rs | 4 +- .../services/shared-sequencer/src/service.rs | 2 +- .../src/selection_algorithms/ratio_tip_gas.rs | 4 +- crates/services/txpool_v2/src/service.rs | 2 +- .../services/txpool_v2/src/storage/graph.rs | 14 +- crates/types/src/test_helpers.rs | 167 ++++++++++ tests/Cargo.toml | 2 + tests/tests/rpc.rs | 268 +++++++++------- tests/tests/trigger_integration/interval.rs | 2 +- 50 files changed, 1567 insertions(+), 515 deletions(-) create mode 100644 .github/actions/slack-notify-template/action.yml create mode 100644 crates/fuel-core/src/service/adapters/rpc.rs create mode 100644 crates/fuel-core/src/service/adapters/rpc/tests.rs diff --git a/.github/actions/slack-notify-template/action.yml b/.github/actions/slack-notify-template/action.yml new file mode 100644 index 00000000000..4988191d3f6 --- /dev/null +++ b/.github/actions/slack-notify-template/action.yml @@ -0,0 +1,25 @@ +name: Notify Slack on Failure +description: Sends notification to Slack if job fails + +inputs: + github_token: + description: Github Token Secret + required: true + slack_webhook: + description: Slack webhook URL + required: true + +runs: + using: composite + steps: + - name: Notify if Job Fails + uses: ravsamhq/notify-slack-action@v2 + with: + status: ${{ job.status }} + token: ${{ inputs.github_token }} + notification_title: '{workflow} has {status_message}' + message_format: '{emoji} *{workflow}* {status_message} in <{repo_url}|{repo}> : <{run_url}|View Run Results>' + footer: '' + notify_when: failure + env: + SLACK_WEBHOOK_URL: ${{ inputs.slack_webhook }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 24712adebb8..9a2f4d399a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,7 +80,7 @@ jobs: run: cargo install cargo-sort - name: Run Cargo.toml sort check run: cargo sort -w --check - - uses: FuelLabs/.github/.github/actions/slack-notify-template@master + - uses: ./.github/actions/slack-notify-template if: always() && github.ref == 'refs/heads/master' with: github_token: ${{ secrets.GITHUB_TOKEN }} @@ -93,7 +93,7 @@ jobs: # ensure openssl hasn't crept into the dep tree - name: Check if openssl is included run: ./.github/workflows/scripts/verify_openssl.sh - - uses: FuelLabs/.github/.github/actions/slack-notify-template@master + - uses: ./.github/actions/slack-notify-template if: always() && github.ref == 'refs/heads/master' with: github_token: ${{ secrets.GITHUB_TOKEN }} @@ -192,7 +192,7 @@ jobs: continue-on-error: true - name: ${{ matrix.command }} ${{ matrix.args }} run: ${{ matrix.env }} cargo ${{ matrix.command }} ${{ matrix.args }} - - uses: FuelLabs/.github/.github/actions/slack-notify-template@master + - uses: ./.github/actions/slack-notify-template if: always() && github.ref == 'refs/heads/master' with: github_token: ${{ secrets.GITHUB_TOKEN }} @@ -220,9 +220,6 @@ jobs: AWS_ACCESS_KEY_ID: test AWS_SECRET_ACCESS_KEY: test AWS_REGION: us-east-1 - AWS_BUCKET: test-bucket - AWS_ENDPOINT_URL: http://localhost:4566 - BUCKET_URL_BASE: test-url.com RUSTFLAGS: -D warnings steps: @@ -339,7 +336,7 @@ jobs: publish-delay: 60000 registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }} - - uses: FuelLabs/.github/.github/actions/slack-notify-template@master + - uses: ./.github/actions/slack-notify-template if: always() with: github_token: ${{ secrets.GITHUB_TOKEN }} @@ -485,7 +482,7 @@ jobs: asset_name: ${{ env.ZIP_FILE_NAME }} asset_content_type: application/gzip - - uses: FuelLabs/.github/.github/actions/slack-notify-template@master + - uses: ./.github/actions/slack-notify-template if: always() && (github.ref == 'refs/heads/master' || github.ref_type == 'tag') && matrix.job.os != 'macos-latest' with: github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml index 78f58d37ffc..eb2e6322614 100644 --- a/.github/workflows/docker-images.yml +++ b/.github/workflows/docker-images.yml @@ -290,7 +290,7 @@ jobs: cache-from: type=registry,ref=${{ env.REGISTRY_URL }}-build-cache-debug:latest cache-to: type=registry,ref=${{ env.REGISTRY_URL }}-build-cache-debug:latest,mode=max,image-manifest=true,oci-mediatypes=true - - uses: FuelLabs/.github/.github/actions/slack-notify-template@master + - uses: ./.github/actions/slack-notify-template if: always() && (github.ref == 'refs/heads/master' || github.ref_type == 'tag') with: github_token: ${{ secrets.GITHUB_TOKEN }} @@ -390,7 +390,7 @@ jobs: cache-from: type=registry,ref=${{ env.REGISTRY_URL }}-build-cache-e2e:latest cache-to: type=registry,ref=${{ env.REGISTRY_URL }}-build-cache-e2e:latest,mode=max,image-manifest=true,oci-mediatypes=true - - uses: FuelLabs/.github/.github/actions/slack-notify-template@master + - uses: ./.github/actions/slack-notify-template if: always() && (github.ref == 'refs/heads/master' || github.ref_type == 'tag') with: github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/Cargo.lock b/Cargo.lock index e0b76069dc2..d63ea62f43d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1290,9 +1290,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "aws-config" -version = "1.8.8" +version = "1.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37cf2b6af2a95a20e266782b4f76f1a5e12bf412a9db2de9c1e9123b9d8c0ad8" +checksum = "1856b1b48b65f71a4dd940b1c0931f9a7b646d4a924b9828ffefc1454714668a" dependencies = [ "aws-credential-types", "aws-runtime", @@ -1436,9 +1436,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.86.0" +version = "1.88.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a0abbfab841446cce6e87af853a3ba2cc1bc9afcd3f3550dd556c43d434c86d" +checksum = "d05b276777560aa9a196dbba2e3aada4d8006d3d7eeb3ba7fe0c317227d933c4" dependencies = [ "aws-credential-types", "aws-runtime", @@ -1458,9 +1458,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.88.0" +version = "1.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a68d675582afea0e94d38b6ca9c5aaae4ca14f1d36faa6edb19b42e687e70d7" +checksum = "f9be14d6d9cd761fac3fd234a0f47f7ed6c0df62d83c0eeb7012750e4732879b" dependencies = [ "aws-credential-types", "aws-runtime", @@ -1480,9 +1480,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.88.0" +version = "1.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d30990923f4f675523c51eb1c0dec9b752fb267b36a61e83cbc219c9d86da715" +checksum = "98a862d704c817d865c8740b62d8bbeb5adcb30965e93b471df8a5bcefa20a80" dependencies = [ "aws-credential-types", "aws-runtime", @@ -3983,6 +3983,16 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "flate2" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "flex-error" version = "0.4.4" @@ -4251,10 +4261,12 @@ version = "0.47.1" dependencies = [ "anyhow", "async-trait", + "aws-config", "aws-sdk-s3", "aws-smithy-mocks", "bytes", "enum-iterator", + "flate2", "fuel-core-protobuf", "fuel-core-services", "fuel-core-storage", @@ -4623,9 +4635,9 @@ dependencies = [ [[package]] name = "fuel-core-protobuf" -version = "0.2.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3beed0ad34ae2f150f597a348e997a17b4991d32d002cd2de7c8e8f931bcc1b" +checksum = "a616726038fbe445bd3294d2700afa8487e38fbc6abc86a8af12be4b596db598" dependencies = [ "prost 0.14.1", "serde", @@ -4791,6 +4803,7 @@ dependencies = [ "aws-sdk-s3", "clap", "cynic", + "flate2", "fuel-core", "fuel-core-benches", "fuel-core-bin", @@ -7256,6 +7269,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ "adler2", + "simd-adler32", ] [[package]] @@ -9842,6 +9856,12 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "similar" version = "2.7.0" diff --git a/Cargo.toml b/Cargo.toml index d012cba82ba..09ff8ca2d26 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -75,6 +75,9 @@ async-graphql = { version = "=7.0.15", features = [ ], default-features = false } async-graphql-value = { version = "=7.0.15" } async-trait = "0.1" + +# Fuel dependencies +aws-config = { version = "1.1.7", features = ["behavior-version-latest"] } aws-sdk-kms = "1.37" axum = "0.5" bytes = "1.5.0" @@ -91,6 +94,7 @@ educe = { version = "0.6", default-features = false, features = [ ] } enum-iterator = "1.2" enum_dispatch = "0.3.13" +flate2 = "1.1.5" fuel-core = { version = "0.47.1", path = "./crates/fuel-core", default-features = false } fuel-core-bin = { version = "0.47.1", path = "./bin/fuel-core" } # Workspace members @@ -110,7 +114,7 @@ fuel-core-p2p = { version = "0.47.1", path = "./crates/services/p2p" } fuel-core-parallel-executor = { version = "0.47.1", path = "./crates/services/parallel-executor" } fuel-core-poa = { version = "0.47.1", path = "./crates/services/consensus_module/poa" } fuel-core-producer = { version = "0.47.1", path = "./crates/services/producer" } -fuel-core-protobuf = { version = "0.2.0" } +fuel-core-protobuf = { version = "0.4.0" } fuel-core-provider = { version = "0.47.1", path = "./crates/provider" } fuel-core-relayer = { version = "0.47.1", path = "./crates/services/relayer" } fuel-core-services = { version = "0.47.1", path = "./crates/services" } @@ -125,8 +129,6 @@ fuel-core-types = { version = "0.47.1", path = "./crates/types", default-feature fuel-core-upgradable-executor = { version = "0.47.1", path = "./crates/services/upgradable-executor" } fuel-core-wasm-executor = { version = "0.47.1", path = "./crates/services/upgradable-executor/wasm-executor", default-features = false } fuel-gas-price-algorithm = { version = "0.47.1", path = "crates/fuel-gas-price-algorithm" } - -# Fuel dependencies fuel-vm-private = { version = "0.65.0", package = "fuel-vm", default-features = false } futures = "0.3" hex = { version = "0.4", features = ["serde"] } diff --git a/bin/fuel-core/src/cli/run.rs b/bin/fuel-core/src/cli/run.rs index dbd8eb5d9cf..c229ed0cb2f 100644 --- a/bin/fuel-core/src/cli/run.rs +++ b/bin/fuel-core/src/cli/run.rs @@ -107,6 +107,7 @@ use std::num::NonZeroUsize; #[cfg(feature = "p2p")] mod p2p; +#[cfg(feature = "rpc")] mod rpc; #[cfg(feature = "shared-sequencer")] @@ -293,6 +294,7 @@ pub struct Command { pub p2p_args: p2p::P2PArgs, #[clap(flatten)] + #[cfg(feature = "rpc")] pub rpc_args: rpc::RpcArgs, #[cfg_attr(feature = "p2p", clap(flatten))] @@ -374,6 +376,7 @@ impl Command { relayer_args, #[cfg(feature = "p2p")] p2p_args, + #[cfg(feature = "rpc")] rpc_args, #[cfg(feature = "p2p")] sync_args, @@ -457,6 +460,7 @@ impl Command { .echo_delegation_interval, }; + #[cfg(feature = "rpc")] let rpc_config = rpc_args.into_config(); let trigger: Trigger = poa_trigger.into(); diff --git a/bin/fuel-core/src/cli/run/rpc.rs b/bin/fuel-core/src/cli/run/rpc.rs index 355353b45a2..a367443112f 100644 --- a/bin/fuel-core/src/cli/run/rpc.rs +++ b/bin/fuel-core/src/cli/run/rpc.rs @@ -1,4 +1,7 @@ -use clap::Args; +use clap::{ + Args, + Subcommand, +}; use fuel_core_types::fuel_types::BlockHeight; use std::net; @@ -11,6 +14,22 @@ pub struct RpcArgs { /// The port to bind the RPC service to #[clap(long = "rpc_port", default_value = "4001", env)] pub rpc_port: u16, + + #[command(subcommand)] + pub storage_method: Option, +} + +#[derive(Debug, Clone, Subcommand)] +pub enum StorageMethod { + Local, + S3 { + #[clap(long = "bucket", env)] + bucket: String, + #[clap(long = "endpoint_url", env)] + endpoint_url: Option, + #[clap(long = "requester_pays", env, default_value = "false")] + requester_pays: bool, + }, } impl RpcArgs { @@ -18,6 +37,26 @@ impl RpcArgs { fuel_core_block_aggregator_api::integration::Config { addr: net::SocketAddr::new(self.rpc_ip, self.rpc_port), sync_from: Some(BlockHeight::from(0)), + storage_method: self.storage_method.map(Into::into).unwrap_or_default(), + } + } +} + +impl From for fuel_core_block_aggregator_api::integration::StorageMethod { + fn from(storage_method: StorageMethod) -> Self { + match storage_method { + StorageMethod::Local => { + fuel_core_block_aggregator_api::integration::StorageMethod::Local + } + StorageMethod::S3 { + bucket, + endpoint_url, + requester_pays, + } => fuel_core_block_aggregator_api::integration::StorageMethod::S3 { + bucket, + endpoint_url, + requester_pays, + }, } } } diff --git a/bin/fuel-core/src/lib.rs b/bin/fuel-core/src/lib.rs index 704f0f3fa5b..67b5193783f 100644 --- a/bin/fuel-core/src/lib.rs +++ b/bin/fuel-core/src/lib.rs @@ -6,4 +6,5 @@ pub mod cli; pub use fuel_core::service::FuelService; +use fuel_core_block_aggregator_api as _; use tikv_jemallocator as _; // Used only by the binary diff --git a/crates/fuel-core/Cargo.toml b/crates/fuel-core/Cargo.toml index 808d8b5920c..6a00676b342 100644 --- a/crates/fuel-core/Cargo.toml +++ b/crates/fuel-core/Cargo.toml @@ -42,6 +42,7 @@ fault-proving = [ "fuel-core-executor/fault-proving", "fuel-core-storage/fault-proving", "fuel-core-chain-config/fault-proving", + "fuel-core-block-aggregator-api/fault-proving", "fuel-core-database/fault-proving", "fuel-core-sync?/fault-proving", "fuel-core-importer/fault-proving", @@ -126,7 +127,7 @@ fuel-core-executor = { workspace = true, features = [ fuel-core-services = { path = "./../services", features = ["test-helpers"] } fuel-core-storage = { path = "./../storage", features = ["test-helpers"] } fuel-core-trace = { path = "./../trace" } -fuel-core-types = { path = "./../types", features = ["test-helpers"] } +fuel-core-types = { path = "./../types", features = ["test-helpers", "random"] } fuel-core-upgradable-executor = { workspace = true, features = [ "test-helpers", ] } diff --git a/crates/fuel-core/src/query/message.rs b/crates/fuel-core/src/query/message.rs index ce8da628f6f..106bedcdb0c 100644 --- a/crates/fuel-core/src/query/message.rs +++ b/crates/fuel-core/src/query/message.rs @@ -313,7 +313,7 @@ fn message_receipts_proof( return Err(anyhow::anyhow!( "Unable to generate the Merkle proof for the message from its receipts" ) - .into()); + .into()) }; // Return the proof. diff --git a/crates/fuel-core/src/schema/tx/assemble_tx.rs b/crates/fuel-core/src/schema/tx/assemble_tx.rs index c73f67cd6c6..84033001ee5 100644 --- a/crates/fuel-core/src/schema/tx/assemble_tx.rs +++ b/crates/fuel-core/src/schema/tx/assemble_tx.rs @@ -637,10 +637,10 @@ where if *amount == 0 { self.tx.outputs_mut().pop(); } else { - break; + break } } else { - break; + break } } } @@ -852,7 +852,7 @@ where } if contracts_not_in_inputs.is_empty() { - break; + break } for contract_id in contracts_not_in_inputs { @@ -913,13 +913,13 @@ where for input in self.tx.inputs() { if input_is_spendable_as_fee(input) { let Some(amount) = input.amount() else { - continue; + continue }; let Some(asset_id) = input.asset_id(&base_asset_id) else { - continue; + continue }; let Some(owner) = input.input_owner() else { - continue; + continue }; if asset_id == &base_asset_id && &fee_payer_account.owner() == owner { @@ -949,7 +949,7 @@ where let need_to_cover = final_fee.saturating_add(self.base_asset_reserved); if need_to_cover <= total_base_asset { - break; + break } let remaining_input_slots = self.remaining_input_slots()?; @@ -1021,7 +1021,7 @@ where for item in items { let key = extractor(item); if !duplicates.insert(key) { - return true + return true; } } diff --git a/crates/fuel-core/src/service.rs b/crates/fuel-core/src/service.rs index a938da1dd94..11bea62351a 100644 --- a/crates/fuel-core/src/service.rs +++ b/crates/fuel-core/src/service.rs @@ -553,7 +553,7 @@ mod tests { service.start_and_await().await.unwrap(); sleep(Duration::from_secs(1)); for service in service.sub_services() { - assert_eq!(service.state(), State::Started); + assert_eq!(service.state(), State::Started,); } if i < service.sub_services().len() { diff --git a/crates/fuel-core/src/service/adapters.rs b/crates/fuel-core/src/service/adapters.rs index d697a8d344b..1abb981ffa7 100644 --- a/crates/fuel-core/src/service/adapters.rs +++ b/crates/fuel-core/src/service/adapters.rs @@ -80,12 +80,15 @@ pub mod fuel_gas_price_provider; pub mod gas_price_adapters; pub mod graphql_api; pub mod import_result_provider; + #[cfg(feature = "p2p")] pub mod p2p; pub mod producer; pub mod ready_signal; #[cfg(feature = "relayer")] pub mod relayer; +#[cfg(feature = "rpc")] +pub mod rpc; #[cfg(feature = "shared-sequencer")] pub mod shared_sequencer; #[cfg(feature = "p2p")] diff --git a/crates/fuel-core/src/service/adapters/rpc.rs b/crates/fuel-core/src/service/adapters/rpc.rs new file mode 100644 index 00000000000..13395b56c46 --- /dev/null +++ b/crates/fuel-core/src/service/adapters/rpc.rs @@ -0,0 +1,53 @@ +use crate::{ + database::{ + Database, + database_description::off_chain::OffChain, + }, + fuel_core_graphql_api::storage::transactions::TransactionStatuses, +}; +use fuel_core_block_aggregator_api::{ + blocks::importer_and_db_source::sync_service::TxReceipts, + result::{ + Error as RPCError, + Result as RPCResult, + }, +}; +use fuel_core_storage::StorageInspect; +use fuel_core_types::{ + fuel_tx::{ + Receipt, + TxId, + }, + services::transaction_status::TransactionExecutionStatus, +}; + +pub struct ReceiptSource { + off_chain: Database, +} + +impl ReceiptSource { + pub fn new(off_chain: Database) -> Self { + Self { off_chain } + } +} + +impl TxReceipts for ReceiptSource { + async fn get_receipts(&self, tx_id: &TxId) -> RPCResult> { + let tx_status = + StorageInspect::::get(&self.off_chain, tx_id) + .map_err(RPCError::receipt_error)?; + if let Some(status) = tx_status { + match status.into_owned() { + TransactionExecutionStatus::Success { receipts, .. } => { + Ok(receipts.to_vec()) + } + _ => Ok(Vec::new()), + } + } else { + Ok(Vec::new()) + } + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/fuel-core/src/service/adapters/rpc/tests.rs b/crates/fuel-core/src/service/adapters/rpc/tests.rs new file mode 100644 index 00000000000..d3065e89eab --- /dev/null +++ b/crates/fuel-core/src/service/adapters/rpc/tests.rs @@ -0,0 +1,46 @@ +#![allow(non_snake_case)] + +use super::*; +use fuel_core_storage::{ + StorageMutate, + transactional::WriteTransaction, +}; +use rand::{ + Rng, + SeedableRng, + prelude::StdRng, +}; +use std::sync::Arc; + +#[tokio::test] +async fn get_receipt__gets_the_receipt_for_expected_tx() { + let mut rng = StdRng::seed_from_u64(9999); + + // given + let mut db = Database::in_memory(); + let tx_id = rng.r#gen(); + let expected = vec![Receipt::Return { + id: rng.r#gen(), + val: 987, + pc: 123, + is: 456, + }]; + let status = TransactionExecutionStatus::Success { + block_height: Default::default(), + time: fuel_core_types::tai64::Tai64(123u64), + result: None, + receipts: Arc::new(expected.clone()), + total_gas: 0, + total_fee: 0, + }; + let mut tx = db.write_transaction(); + StorageMutate::::insert(&mut tx, &tx_id, &status).unwrap(); + tx.commit().unwrap(); + let receipt_source = ReceiptSource::new(db); + + // when + let actual = receipt_source.get_receipts(&tx_id).await.unwrap(); + + // then + assert_eq!(actual, expected); +} diff --git a/crates/fuel-core/src/service/config.rs b/crates/fuel-core/src/service/config.rs index 2bc98848e1c..2200818018d 100644 --- a/crates/fuel-core/src/service/config.rs +++ b/crates/fuel-core/src/service/config.rs @@ -53,6 +53,8 @@ use fuel_core_types::fuel_types::{ ChainId, }; +#[cfg(feature = "rpc")] +use fuel_core_block_aggregator_api::integration::StorageMethod; #[cfg(feature = "parallel-executor")] use std::num::NonZeroUsize; @@ -176,6 +178,7 @@ impl Config { let rpc_config = fuel_core_block_aggregator_api::integration::Config { addr: free_local_addr(), sync_from: Some(BlockHeight::from(0)), + storage_method: StorageMethod::Local, }; Self { diff --git a/crates/fuel-core/src/service/sub_services.rs b/crates/fuel-core/src/service/sub_services.rs index 12160c217d4..d7aa20ef95a 100644 --- a/crates/fuel-core/src/service/sub_services.rs +++ b/crates/fuel-core/src/service/sub_services.rs @@ -22,6 +22,8 @@ use crate::service::adapters::consensus_module::poa::pre_confirmation_signature: trigger::TimeBasedTrigger, tx_receiver::PreconfirmationsReceiver, }; +#[cfg(feature = "rpc")] +use crate::service::adapters::rpc::ReceiptSource; use crate::{ combined_database::CombinedDatabase, database::Database, @@ -66,8 +68,8 @@ use anyhow::anyhow; use fuel_core_block_aggregator_api::{ blocks::importer_and_db_source::serializer_adapter::SerializerAdapter, db::storage_or_remote_db::StorageOrRemoteDB, - db::storage_or_remote_db::get_env_vars, db::table::LatestBlock, + integration::StorageMethod, result::Error, }; use fuel_core_compression_service::service::new_service as new_compression_service; @@ -91,7 +93,6 @@ use fuel_core_storage::{ Error as StorageError, StorageAsRef, }; - #[cfg(feature = "relayer")] use fuel_core_types::blockchain::primitives::DaBlockHeight; use fuel_core_types::signer::SignMode; @@ -473,54 +474,41 @@ pub fn init_sub_services( let block_aggregator_config = config.rpc_config.clone(); let sync_from = block_aggregator_config.sync_from.unwrap_or_default(); let sync_from_height; - let db_adapter = if let Some(( - aws_access_key_id, - aws_secrete_access_key, - aws_region, - aws_bucket, - url_base, - aws_endpoint_url, - )) = get_env_vars() - { - let db = database.block_aggregation_s3().clone(); - let maybe_sync_from_height = db - .storage_as_ref::() - .get(&()) - .map_err(|e: StorageError| Error::DB(anyhow!(e)))? - .map(|c| *c) - .and_then(|h| h.succ()); - sync_from_height = maybe_sync_from_height.unwrap_or(sync_from); - - StorageOrRemoteDB::new_s3( - db, - &aws_access_key_id, - &aws_secrete_access_key, - &aws_region, - &aws_bucket, - &url_base, - aws_endpoint_url, - sync_from, - ) - } else { - tracing::info!( - "Required environment variables for S3 bucket not set. Requires: \n\ - AWS_ACCESS_KEY_ID \n\ - AWS_SECRET_ACCESS_KEY \n\ - AWS_REGION \n\ - AWS_BUCKET \n\ - AWS_ENDPOINT_URL \n\ - AWS_S3_URL_BASE (Optional)\n\ - Using local storage" - ); - let db = database.block_aggregation_storage().clone(); - let maybe_sync_from_height = db - .storage_as_ref::() - .get(&()) - .map_err(|e: StorageError| Error::DB(anyhow!(e)))? - .map(|c| *c) - .and_then(|h| h.succ()); - sync_from_height = maybe_sync_from_height.unwrap_or(sync_from); - StorageOrRemoteDB::new_storage(db, sync_from) + let receipts = ReceiptSource::new(database.off_chain().clone()); + let db_adapter = match &block_aggregator_config.storage_method { + StorageMethod::Local => { + let db = database.block_aggregation_storage().clone(); + let maybe_sync_from_height = db + .storage_as_ref::() + .get(&()) + .map_err(|e: StorageError| Error::DB(anyhow!(e)))? + .map(|c| *c) + .and_then(|h| h.succ()); + sync_from_height = maybe_sync_from_height.unwrap_or(sync_from); + StorageOrRemoteDB::new_storage(db, sync_from) + } + StorageMethod::S3 { + bucket, + endpoint_url, + requester_pays, + } => { + let db = database.block_aggregation_s3().clone(); + let maybe_sync_from_height = db + .storage_as_ref::() + .get(&()) + .map_err(|e: StorageError| Error::DB(anyhow!(e)))? + .map(|c| *c) + .and_then(|h| h.succ()); + sync_from_height = maybe_sync_from_height.unwrap_or(sync_from); + + StorageOrRemoteDB::new_s3( + db, + bucket, + *requester_pays, + endpoint_url.clone(), + sync_from, + ) + } }; let serializer = SerializerAdapter; let onchain_db = database.on_chain().clone(); @@ -530,6 +518,7 @@ pub fn init_sub_services( db_adapter, serializer, onchain_db, + receipts, importer, sync_from_height, )? diff --git a/crates/services/block_aggregator_api/Cargo.toml b/crates/services/block_aggregator_api/Cargo.toml index 0a94653bc63..85112fdfc4d 100644 --- a/crates/services/block_aggregator_api/Cargo.toml +++ b/crates/services/block_aggregator_api/Cargo.toml @@ -15,10 +15,12 @@ fault-proving = ["fuel-core-types/fault-proving"] [dependencies] anyhow = { workspace = true } async-trait = { workspace = true } +aws-config = { workspace = true } aws-sdk-s3 = "1.111.0" aws-smithy-mocks = "0.2.0" bytes = { workspace = true, features = ["serde"] } enum-iterator = { workspace = true } +flate2 = { workspace = true } fuel-core-protobuf = { workspace = true } fuel-core-services = { workspace = true } fuel-core-storage = { workspace = true, features = ["std"] } diff --git a/crates/services/block_aggregator_api/src/api.rs b/crates/services/block_aggregator_api/src/api.rs index 20d6d58003d..0e07b31b0bb 100644 --- a/crates/services/block_aggregator_api/src/api.rs +++ b/crates/services/block_aggregator_api/src/api.rs @@ -29,7 +29,7 @@ pub enum BlockAggregatorQuery { }, // TODO: Do we need a way to unsubscribe or can we just see that the receiver is dropped? NewBlockSubscription { - response: tokio::sync::mpsc::Sender, + response: tokio::sync::mpsc::Sender<(BlockHeight, Block)>, }, } @@ -75,7 +75,8 @@ impl BlockAggregatorQuery { (query, receiver) } - pub fn new_block_subscription() -> (Self, tokio::sync::mpsc::Receiver) { + pub fn new_block_subscription() + -> (Self, tokio::sync::mpsc::Receiver<(BlockHeight, B)>) { const ARBITRARY_CHANNEL_SIZE: usize = 10; let (sender, receiver) = tokio::sync::mpsc::channel(ARBITRARY_CHANNEL_SIZE); let query = Self::NewBlockSubscription { response: sender }; diff --git a/crates/services/block_aggregator_api/src/api/protobuf_adapter.rs b/crates/services/block_aggregator_api/src/api/protobuf_adapter.rs index d7c25937841..0c0df12ffbe 100644 --- a/crates/services/block_aggregator_api/src/api/protobuf_adapter.rs +++ b/crates/services/block_aggregator_api/src/api/protobuf_adapter.rs @@ -14,12 +14,14 @@ use crate::{ BlockRangeRequest as ProtoBlockRangeRequest, BlockResponse as ProtoBlockResponse, NewBlockSubscriptionRequest as ProtoNewBlockSubscriptionRequest, - RemoteBlockRangeResponse as ProtoRemoteBlockRangeResponse, + RemoteBlockResponse as ProtoRemoteBlockResponse, + RemoteS3Bucket as ProtoRemoteS3Bucket, block_aggregator_server::{ BlockAggregator, BlockAggregatorServer as ProtoBlockAggregatorServer, }, block_response as proto_block_response, + remote_block_response::Location as ProtoRemoteLocation, }, result::{ Error, @@ -60,7 +62,7 @@ impl Server { #[async_trait] impl BlockAggregator for Server { - async fn get_block_height( + async fn get_synced_block_height( &self, request: tonic::Request, ) -> Result, tonic::Status> { @@ -106,8 +108,9 @@ impl BlockAggregator for Server { Ok(block_range_response) => match block_range_response { BlockRangeResponse::Literal(inner) => { let stream = inner - .map(|res| { + .map(|(height, res)| { let response = ProtoBlockResponse { + height: *height, payload: Some(proto_block_response::Payload::Literal( res, )), @@ -117,16 +120,21 @@ impl BlockAggregator for Server { .boxed(); Ok(tonic::Response::new(stream)) } - BlockRangeResponse::Remote(inner) => { + BlockRangeResponse::S3(inner) => { let stream = inner - .map(|res| { - let proto_response = ProtoRemoteBlockRangeResponse { - region: res.region.clone(), - bucket: res.bucket.clone(), - key: res.key.clone(), - url: res.url.clone(), + .map(|(height, res)| { + let s3 = ProtoRemoteS3Bucket { + bucket: res.bucket, + key: res.key, + requester_pays: res.requester_pays, + endpoint: res.aws_endpoint, + }; + let location = ProtoRemoteLocation::S3(s3); + let proto_response = ProtoRemoteBlockResponse { + location: Some(location), }; let response = ProtoBlockResponse { + height: *height, payload: Some(proto_block_response::Payload::Remote( proto_response, )), @@ -161,8 +169,9 @@ impl BlockAggregator for Server { let (task_sender, task_receiver) = tokio::sync::mpsc::channel(ARB_CHANNEL_SIZE); tokio::spawn(async move { - while let Some(nb) = receiver.recv().await { + while let Some((height, nb)) = receiver.recv().await { let response = ProtoBlockResponse { + height: *height, payload: Some(proto_block_response::Payload::Literal(nb)), }; if task_sender.send(Ok(response)).await.is_err() { diff --git a/crates/services/block_aggregator_api/src/api/protobuf_adapter/tests.rs b/crates/services/block_aggregator_api/src/api/protobuf_adapter/tests.rs index 6940173fd27..111cf1d303f 100644 --- a/crates/services/block_aggregator_api/src/api/protobuf_adapter/tests.rs +++ b/crates/services/block_aggregator_api/src/api/protobuf_adapter/tests.rs @@ -8,7 +8,7 @@ use crate::{ }, block_range_response::{ BlockRangeResponse, - RemoteBlockRangeResponse, + RemoteS3Response, }, blocks::importer_and_db_source::{ BlockSerializer, @@ -26,6 +26,7 @@ use crate::{ block_response::Payload, }, }; +use fuel_core_protobuf::remote_block_response::Location; use fuel_core_types::{ blockchain::block::Block as FuelBlock, fuel_types::BlockHeight, @@ -56,7 +57,7 @@ async fn await_query__get_current_height__client_receives_expected_value() { let handle = tokio::spawn(async move { tracing::info!("querying with client"); client - .get_block_height(BlockHeightRequest {}) + .get_synced_block_height(BlockHeightRequest {}) .await .expect("could not get height") }); @@ -107,15 +108,16 @@ async fn await_query__get_block_range__client_receives_expected_value__literal() let serializer_adapter = SerializerAdapter; let fuel_block_1 = FuelBlock::default(); let mut fuel_block_2 = FuelBlock::default(); - let block_height_2 = fuel_block_1.header().height().succ().unwrap(); + let block_height_1 = fuel_block_1.header().height(); + let block_height_2 = block_height_1.succ().unwrap(); fuel_block_2.header_mut().set_block_height(block_height_2); let block1 = serializer_adapter - .serialize_block(&fuel_block_1) + .serialize_block(&fuel_block_1, &[]) .expect("could not serialize block"); let block2 = serializer_adapter - .serialize_block(&fuel_block_2) + .serialize_block(&fuel_block_2, &[]) .expect("could not serialize block"); - let list = vec![block1, block2]; + let list = vec![(*block_height_1, block1), (block_height_2, block2)]; // return response through query's channel if let BlockAggregatorQuery::GetBlockRange { first, @@ -135,7 +137,7 @@ async fn await_query__get_block_range__client_receives_expected_value__literal() tracing::info!("awaiting query"); let response = handle.await.unwrap(); let expected = list; - let actual: Vec = response + let actual: Vec<(BlockHeight, ProtoBlock)> = response .into_inner() .try_collect::>() .await @@ -143,7 +145,7 @@ async fn await_query__get_block_range__client_receives_expected_value__literal() .into_iter() .map(|b| { if let Some(Payload::Literal(inner)) = b.payload { - inner + (BlockHeight::new(b.height), inner) } else { panic!("unexpected response type") } @@ -178,19 +180,18 @@ async fn await_query__get_block_range__client_receives_expected_value__remote() let query = api.await_query().await.unwrap(); // then - let list: Vec<_> = ["1", "2"] + let list: Vec<_> = [(BlockHeight::new(1), "1"), (BlockHeight::new(2), "2")] .iter() - .map(|height| { - let region = "test-region".to_string(); + .map(|(height, key)| { let bucket = "test-bucket".to_string(); - let key = height.to_string(); - let url = "good.url".to_string(); - RemoteBlockRangeResponse { - region, + let key = key.to_string(); + let res = RemoteS3Response { bucket, key, - url, - } + requester_pays: false, + aws_endpoint: None, + }; + (*height, res) }) .collect(); // return response through query's channel @@ -204,7 +205,7 @@ async fn await_query__get_block_range__client_receives_expected_value__remote() assert_eq!(last, BlockHeight::new(1)); tracing::info!("correct query received, sending response"); let stream = tokio_stream::iter(list.clone()).boxed(); - let range = BlockRangeResponse::Remote(stream); + let range = BlockRangeResponse::S3(stream); response.send(range).unwrap(); } else { panic!("expected GetBlockRange query"); @@ -212,7 +213,7 @@ async fn await_query__get_block_range__client_receives_expected_value__remote() tracing::info!("awaiting query"); let response = handle.await.unwrap(); let expected = list; - let actual: Vec = response + let actual: Vec<(BlockHeight, RemoteS3Response)> = response .into_inner() .try_collect::>() .await @@ -220,12 +221,18 @@ async fn await_query__get_block_range__client_receives_expected_value__remote() .into_iter() .map(|b| { if let Some(Payload::Remote(inner)) = b.payload { - RemoteBlockRangeResponse { - region: inner.region, - bucket: inner.bucket, - key: inner.key, - url: inner.url, - } + let height = BlockHeight::new(b.height); + let location = inner.location.unwrap(); + let Location::S3(s3) = location else { + panic!("unexpected location type") + }; + let res = RemoteS3Response { + bucket: s3.bucket, + key: s3.key, + requester_pays: false, + aws_endpoint: None, + }; + (height, res) } else { panic!("unexpected response type") } @@ -269,16 +276,16 @@ async fn await_query__new_block_stream__client_receives_expected_value() { let mut fuel_block_2 = FuelBlock::default(); fuel_block_2.header_mut().set_block_height(height2); let block1 = serializer_adapter - .serialize_block(&fuel_block_1) + .serialize_block(&fuel_block_1, &[]) .expect("could not serialize block"); let block2 = serializer_adapter - .serialize_block(&fuel_block_2) + .serialize_block(&fuel_block_2, &[]) .expect("could not serialize block"); - let list = vec![block1, block2]; + let list = vec![(height1, block1), (height2, block2)]; if let BlockAggregatorQuery::NewBlockSubscription { response } = query { tracing::info!("correct query received, sending response"); - for block in list.clone() { - response.send(block).await.unwrap(); + for (height, block) in list.clone() { + response.send((height, block)).await.unwrap(); } } else { panic!("expected GetBlockRange query"); @@ -286,7 +293,7 @@ async fn await_query__new_block_stream__client_receives_expected_value() { tracing::info!("awaiting query"); let response = handle.await.unwrap(); let expected = list; - let actual: Vec = response + let actual: Vec<(BlockHeight, ProtoBlock)> = response .into_inner() .try_collect::>() .await @@ -294,7 +301,7 @@ async fn await_query__new_block_stream__client_receives_expected_value() { .into_iter() .map(|b| { if let Some(Payload::Literal(inner)) = b.payload { - inner + (BlockHeight::new(b.height), inner) } else { panic!("unexpected response type") } diff --git a/crates/services/block_aggregator_api/src/block_aggregator.rs b/crates/services/block_aggregator_api/src/block_aggregator.rs index 42ff7ecd16c..48009d6cfa0 100644 --- a/crates/services/block_aggregator_api/src/block_aggregator.rs +++ b/crates/services/block_aggregator_api/src/block_aggregator.rs @@ -100,7 +100,7 @@ where async fn handle_new_block_subscription( &mut self, - response: tokio::sync::mpsc::Sender, + response: tokio::sync::mpsc::Sender<(BlockHeight, Blocks::Block)>, ) -> TaskNextAction { self.new_block_subscriptions.push(response); TaskNextAction::Continue @@ -120,7 +120,7 @@ where match &event { BlockSourceEvent::NewBlock(height, block) => { self.new_block_subscriptions.retain_mut(|sub| { - let send_res = sub.try_send(block.clone()); + let send_res = sub.try_send((*height, block.clone())); match send_res { Ok(_) => true, Err(tokio::sync::mpsc::error::TrySendError::Full(_)) => { diff --git a/crates/services/block_aggregator_api/src/block_range_response.rs b/crates/services/block_aggregator_api/src/block_range_response.rs index 6a80da26ac8..76d05465906 100644 --- a/crates/services/block_aggregator_api/src/block_range_response.rs +++ b/crates/services/block_aggregator_api/src/block_range_response.rs @@ -1,22 +1,23 @@ use crate::protobuf_types::Block as ProtoBlock; use fuel_core_services::stream::Stream; +use fuel_core_types::fuel_types::BlockHeight; pub type BoxStream = core::pin::Pin + Send + 'static>>; /// The response to a block range query, either as a literal stream of blocks or as a remote URL pub enum BlockRangeResponse { /// A literal stream of blocks - Literal(BoxStream), + Literal(BoxStream<(BlockHeight, ProtoBlock)>), /// A remote URL where the blocks can be fetched - Remote(BoxStream), + S3(BoxStream<(BlockHeight, RemoteS3Response)>), } #[derive(Debug, Clone, PartialEq, Eq)] -pub struct RemoteBlockRangeResponse { - pub region: String, +pub struct RemoteS3Response { pub bucket: String, pub key: String, - pub url: String, + pub requester_pays: bool, + pub aws_endpoint: Option, } #[cfg(test)] @@ -24,7 +25,7 @@ impl std::fmt::Debug for BlockRangeResponse { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { BlockRangeResponse::Literal(_) => f.debug_struct("Literal").finish(), - BlockRangeResponse::Remote(_url) => f.debug_struct("Remote").finish(), + BlockRangeResponse::S3(_url) => f.debug_struct("Remote").finish(), } } } diff --git a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source.rs b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source.rs index 3bf49003ccc..872bbb74a43 100644 --- a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source.rs +++ b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source.rs @@ -16,6 +16,7 @@ use fuel_core_services::{ stream::BoxStream, }; use fuel_core_storage::{ + Error as StorageError, StorageInspect, tables::FuelBlocks, }; @@ -25,8 +26,12 @@ use fuel_core_types::{ services::block_importer::SharedImportResult, }; -use crate::blocks::importer_and_db_source::sync_service::SyncTask; +use crate::blocks::importer_and_db_source::sync_service::{ + SyncTask, + TxReceipts, +}; use fuel_core_storage::tables::Transactions; +use fuel_core_types::fuel_tx::Receipt as FuelReceipt; pub mod importer_service; pub mod sync_service; @@ -37,41 +42,44 @@ pub mod serializer_adapter; pub trait BlockSerializer { type Block; - fn serialize_block(&self, block: &FuelBlock) -> Result; + fn serialize_block( + &self, + block: &FuelBlock, + receipts: &[FuelReceipt], + ) -> Result; } /// A block source that combines an importer and a database sync task. /// Old blocks will be synced from a target database and new blocks will be received from /// the importer -pub struct ImporterAndDbSource +pub struct ImporterAndDbSource where Serializer: BlockSerializer + Send + Sync + 'static, ::Block: Send + Sync + 'static, DB: Send + Sync + 'static, - DB: StorageInspect, - DB: StorageInspect, - E: std::fmt::Debug + Send, + DB: StorageInspect, + DB: StorageInspect, + Receipts: TxReceipts, { importer_task: ServiceRunner>, - sync_task: ServiceRunner>, + sync_task: ServiceRunner>, /// Receive blocks from the importer and sync tasks receiver: tokio::sync::mpsc::Receiver>, - - _error_marker: std::marker::PhantomData, } -impl ImporterAndDbSource +impl ImporterAndDbSource where Serializer: BlockSerializer + Clone + Send + Sync + 'static, ::Block: Send + Sync + 'static, - DB: StorageInspect + Send + Sync, - DB: StorageInspect + Send + 'static, - E: std::fmt::Debug + Send, + DB: StorageInspect + Send + Sync, + DB: StorageInspect + Send + 'static, + Receipts: TxReceipts, { pub fn new( importer: BoxStream, serializer: Serializer, - database: DB, + db: DB, + receipts: Receipts, db_starting_height: BlockHeight, db_ending_height: BlockHeight, ) -> Self { @@ -84,7 +92,8 @@ where let sync_task = SyncTask::new( serializer, block_return, - database, + db, + receipts, db_starting_height, db_ending_height, ); @@ -94,19 +103,19 @@ where importer_task: importer_runner, sync_task: sync_runner, receiver, - _error_marker: std::marker::PhantomData, } } } -impl BlockSource for ImporterAndDbSource +impl BlockSource + for ImporterAndDbSource where Serializer: BlockSerializer + Send + Sync + 'static, ::Block: Send + Sync + 'static, - DB: Send + Sync, - DB: StorageInspect, - DB: StorageInspect, - E: std::fmt::Debug + Send + Sync, + DB: Send + Sync + 'static, + DB: StorageInspect, + DB: StorageInspect, + Receipts: TxReceipts, { type Block = Serializer::Block; diff --git a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/importer_service.rs b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/importer_service.rs index b939400811c..99721b06ad2 100644 --- a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/importer_service.rs +++ b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/importer_service.rs @@ -69,9 +69,15 @@ where match maybe_import_result { Some(import_result) => { let height = import_result.sealed_block.entity.header().height(); + let receipts = import_result + .tx_status + .iter() + .flat_map(|status| status.result.receipts()) + .map(Clone::clone) + .collect::>(); let res = self .serializer - .serialize_block(&import_result.sealed_block.entity); + .serialize_block(&import_result.sealed_block.entity, &receipts); let block = try_or_continue!(res); let event = BlockSourceEvent::NewBlock(*height, block); let res = self.block_return_sender.send(event).await; diff --git a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter.rs b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter.rs index 04247de5aa6..e24932de4b3 100644 --- a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter.rs +++ b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter.rs @@ -14,7 +14,8 @@ use fuel_core_types::{ block::Block as FuelBlock, }, }; -use crate::blocks::importer_and_db_source::serializer_adapter::fuel_to_proto_conversions::{proto_header_from_header, proto_tx_from_tx}; +use fuel_core_types::fuel_tx::Receipt as FuelReceipt; +use crate::blocks::importer_and_db_source::serializer_adapter::fuel_to_proto_conversions::{proto_header_from_header, proto_receipt_from_receipt, proto_tx_from_tx}; #[derive(Clone)] pub struct SerializerAdapter; @@ -22,7 +23,11 @@ pub struct SerializerAdapter; impl BlockSerializer for SerializerAdapter { type Block = ProtoBlock; - fn serialize_block(&self, block: &FuelBlock) -> crate::result::Result { + fn serialize_block( + &self, + block: &FuelBlock, + receipts: &[FuelReceipt], + ) -> crate::result::Result { let proto_header = proto_header_from_header(block.header()); match &block { FuelBlock::V1(_) => { @@ -33,6 +38,7 @@ impl BlockSerializer for SerializerAdapter { .iter() .map(proto_tx_from_tx) .collect(), + receipts: receipts.iter().map(proto_receipt_from_receipt).collect(), }; Ok(ProtoBlock { versioned_block: Some(ProtoVersionedBlock::V1(proto_v1_block)), @@ -52,27 +58,30 @@ pub mod proto_to_fuel_conversions; #[cfg(test)] mod tests { use super::*; - use fuel_core_types::test_helpers::arb_block; + use fuel_core_types::test_helpers::{arb_block, arb_receipts}; use proptest::prelude::*; use crate::blocks::importer_and_db_source::serializer_adapter::proto_to_fuel_conversions::fuel_block_from_protobuf; proptest! { #![proptest_config(ProptestConfig { - cases: 100, .. ProptestConfig::default() + cases: 1, .. ProptestConfig::default() })] #[test] - fn serialize_block__roundtrip((block, msg_ids, event_inbox_root) in arb_block()) { + fn serialize_block__roundtrip( + (block, msg_ids, event_inbox_root) in arb_block(), + receipts in arb_receipts()) + { // given let serializer = SerializerAdapter; // when - let proto_block = serializer.serialize_block(&block).unwrap(); + let proto_block = serializer.serialize_block(&block, &receipts).unwrap(); // then - let deserialized_block = fuel_block_from_protobuf(proto_block, &msg_ids, event_inbox_root).unwrap(); + let (deserialized_block, deserialized_receipts) = fuel_block_from_protobuf(proto_block, &msg_ids, event_inbox_root).unwrap(); assert_eq!(block, deserialized_block); - - } + assert_eq!(receipts, deserialized_receipts); + } } #[test] diff --git a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/fuel_to_proto_conversions.rs b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/fuel_to_proto_conversions.rs index 8057c6494fe..66c3a84ce29 100644 --- a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/fuel_to_proto_conversions.rs +++ b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/fuel_to_proto_conversions.rs @@ -21,6 +21,7 @@ use crate::{ MintTransaction as ProtoMintTx, Output as ProtoOutput, Policies as ProtoPolicies, + Receipt as ProtoReceipt, ScriptTransaction as ProtoScriptTx, StorageSlot as ProtoStorageSlot, Transaction as ProtoTransaction, @@ -36,6 +37,8 @@ use crate::{ header::VersionedHeader as ProtoVersionedHeader, input::Variant as ProtoInputVariant, output::Variant as ProtoOutputVariant, + receipt::Variant as ProtoReceiptVariant, + script_execution_result::Variant as ProtoScriptExecutionResultVariant, transaction::Variant as ProtoTransactionVariant, upgrade_purpose::Variant as ProtoUpgradePurposeVariant, }, @@ -53,9 +56,12 @@ use fuel_core_types::{ }, primitives::BlockId, }, + fuel_asm::PanicInstruction, fuel_tx::{ Input, Output, + Receipt as FuelReceipt, + ScriptExecutionResult, StorageSlot, Transaction as FuelTransaction, TxPointer, @@ -564,3 +570,286 @@ fn proto_policies_from_policies( values: values.to_vec(), } } + +fn proto_script_execution_result( + result: &ScriptExecutionResult, +) -> crate::protobuf_types::ScriptExecutionResult { + use crate::protobuf_types::{ + ScriptExecutionResult as ProtoScriptExecutionResult, + ScriptExecutionResultGenericFailure as ProtoScriptExecutionResultGenericFailure, + ScriptExecutionResultPanic as ProtoScriptExecutionResultPanic, + ScriptExecutionResultRevert as ProtoScriptExecutionResultRevert, + ScriptExecutionResultSuccess as ProtoScriptExecutionResultSuccess, + }; + + let variant = match result { + ScriptExecutionResult::Success => ProtoScriptExecutionResultVariant::Success( + ProtoScriptExecutionResultSuccess {}, + ), + ScriptExecutionResult::Revert => { + ProtoScriptExecutionResultVariant::Revert(ProtoScriptExecutionResultRevert {}) + } + ScriptExecutionResult::Panic => { + ProtoScriptExecutionResultVariant::Panic(ProtoScriptExecutionResultPanic {}) + } + ScriptExecutionResult::GenericFailure(code) => { + ProtoScriptExecutionResultVariant::GenericFailure( + ProtoScriptExecutionResultGenericFailure { code: *code }, + ) + } + }; + + ProtoScriptExecutionResult { + variant: Some(variant), + } +} + +fn proto_panic_instruction( + panic_instruction: &PanicInstruction, +) -> crate::protobuf_types::PanicInstruction { + use crate::protobuf_types::PanicReason as ProtoPanicReason; + + let reason_value = *panic_instruction.reason() as u8; + let reason = ProtoPanicReason::try_from(i32::from(reason_value)) + .unwrap_or(ProtoPanicReason::Unknown); + + crate::protobuf_types::PanicInstruction { + reason: reason as i32, + instruction: *panic_instruction.instruction(), + } +} + +pub fn proto_receipt_from_receipt(receipt: &FuelReceipt) -> ProtoReceipt { + match receipt { + FuelReceipt::Call { + id, + to, + amount, + asset_id, + gas, + param1, + param2, + pc, + is, + } => ProtoReceipt { + variant: Some(ProtoReceiptVariant::Call( + crate::protobuf_types::CallReceipt { + id: id.as_ref().to_vec(), + to: to.as_ref().to_vec(), + amount: *amount, + asset_id: asset_id.as_ref().to_vec(), + gas: *gas, + param1: *param1, + param2: *param2, + pc: *pc, + is: *is, + }, + )), + }, + FuelReceipt::Return { id, val, pc, is } => ProtoReceipt { + variant: Some(ProtoReceiptVariant::ReturnReceipt( + crate::protobuf_types::ReturnReceipt { + id: id.as_ref().to_vec(), + val: *val, + pc: *pc, + is: *is, + }, + )), + }, + FuelReceipt::ReturnData { + id, + ptr, + len, + digest, + pc, + is, + data, + } => ProtoReceipt { + variant: Some(ProtoReceiptVariant::ReturnData( + crate::protobuf_types::ReturnDataReceipt { + id: id.as_ref().to_vec(), + ptr: *ptr, + len: *len, + digest: digest.as_ref().to_vec(), + pc: *pc, + is: *is, + data: data.as_ref().map(|b| b.to_vec()), + }, + )), + }, + FuelReceipt::Panic { + id, + reason, + pc, + is, + contract_id, + } => ProtoReceipt { + variant: Some(ProtoReceiptVariant::Panic( + crate::protobuf_types::PanicReceipt { + id: id.as_ref().to_vec(), + reason: Some(proto_panic_instruction(reason)), + pc: *pc, + is: *is, + contract_id: contract_id.as_ref().map(|cid| cid.as_ref().to_vec()), + }, + )), + }, + FuelReceipt::Revert { id, ra, pc, is } => ProtoReceipt { + variant: Some(ProtoReceiptVariant::Revert( + crate::protobuf_types::RevertReceipt { + id: id.as_ref().to_vec(), + ra: *ra, + pc: *pc, + is: *is, + }, + )), + }, + FuelReceipt::Log { + id, + ra, + rb, + rc, + rd, + pc, + is, + } => ProtoReceipt { + variant: Some(ProtoReceiptVariant::Log( + crate::protobuf_types::LogReceipt { + id: id.as_ref().to_vec(), + ra: *ra, + rb: *rb, + rc: *rc, + rd: *rd, + pc: *pc, + is: *is, + }, + )), + }, + FuelReceipt::LogData { + id, + ra, + rb, + ptr, + len, + digest, + pc, + is, + data, + } => ProtoReceipt { + variant: Some(ProtoReceiptVariant::LogData( + crate::protobuf_types::LogDataReceipt { + id: id.as_ref().to_vec(), + ra: *ra, + rb: *rb, + ptr: *ptr, + len: *len, + digest: digest.as_ref().to_vec(), + pc: *pc, + is: *is, + data: data.as_ref().map(|b| b.to_vec()), + }, + )), + }, + FuelReceipt::Transfer { + id, + to, + amount, + asset_id, + pc, + is, + } => ProtoReceipt { + variant: Some(ProtoReceiptVariant::Transfer( + crate::protobuf_types::TransferReceipt { + id: id.as_ref().to_vec(), + to: to.as_ref().to_vec(), + amount: *amount, + asset_id: asset_id.as_ref().to_vec(), + pc: *pc, + is: *is, + }, + )), + }, + FuelReceipt::TransferOut { + id, + to, + amount, + asset_id, + pc, + is, + } => ProtoReceipt { + variant: Some(ProtoReceiptVariant::TransferOut( + crate::protobuf_types::TransferOutReceipt { + id: id.as_ref().to_vec(), + to: to.as_ref().to_vec(), + amount: *amount, + asset_id: asset_id.as_ref().to_vec(), + pc: *pc, + is: *is, + }, + )), + }, + FuelReceipt::ScriptResult { result, gas_used } => ProtoReceipt { + variant: Some(ProtoReceiptVariant::ScriptResult( + crate::protobuf_types::ScriptResultReceipt { + result: Some(proto_script_execution_result(result)), + gas_used: *gas_used, + }, + )), + }, + FuelReceipt::MessageOut { + sender, + recipient, + amount, + nonce, + len, + digest, + data, + } => ProtoReceipt { + variant: Some(ProtoReceiptVariant::MessageOut( + crate::protobuf_types::MessageOutReceipt { + sender: sender.as_ref().to_vec(), + recipient: recipient.as_ref().to_vec(), + amount: *amount, + nonce: nonce.as_ref().to_vec(), + len: *len, + digest: digest.as_ref().to_vec(), + data: data.as_ref().map(|b| b.to_vec()), + }, + )), + }, + FuelReceipt::Mint { + sub_id, + contract_id, + val, + pc, + is, + } => ProtoReceipt { + variant: Some(ProtoReceiptVariant::Mint( + crate::protobuf_types::MintReceipt { + sub_id: sub_id.as_ref().to_vec(), + contract_id: contract_id.as_ref().to_vec(), + val: *val, + pc: *pc, + is: *is, + }, + )), + }, + FuelReceipt::Burn { + sub_id, + contract_id, + val, + pc, + is, + } => ProtoReceipt { + variant: Some(ProtoReceiptVariant::Burn( + crate::protobuf_types::BurnReceipt { + sub_id: sub_id.as_ref().to_vec(), + contract_id: contract_id.as_ref().to_vec(), + val: *val, + pc: *pc, + is: *is, + }, + )), + }, + } +} diff --git a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/proto_to_fuel_conversions.rs b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/proto_to_fuel_conversions.rs index 5e69f063f72..bf715b0e30e 100644 --- a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/proto_to_fuel_conversions.rs +++ b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/serializer_adapter/proto_to_fuel_conversions.rs @@ -8,6 +8,7 @@ use crate::{ Header as ProtoHeader, Input as ProtoInput, Output as ProtoOutput, + PanicInstruction as ProtoPanicInstruction, Policies as ProtoPolicies, StorageSlot as ProtoStorageSlot, Transaction as ProtoTransaction, @@ -18,6 +19,8 @@ use crate::{ header::VersionedHeader as ProtoVersionedHeader, input::Variant as ProtoInputVariant, output::Variant as ProtoOutputVariant, + receipt::Variant as ProtoReceiptVariant, + script_execution_result::Variant as ProtoScriptExecutionResultVariant, transaction::Variant as ProtoTransactionVariant, upgrade_purpose::Variant as ProtoUpgradePurposeVariant, }, @@ -37,12 +40,18 @@ use fuel_core_types::{ Empty, }, }, + fuel_asm::{ + PanicInstruction, + PanicReason, + }, fuel_tx::{ Address, BlobBody, Bytes32, Input, Output, + Receipt as FuelReceipt, + ScriptExecutionResult, StorageSlot, Transaction as FuelTransaction, TxPointer, @@ -57,6 +66,12 @@ use fuel_core_types::{ PolicyType, }, }, + fuel_types::{ + AssetId, + ContractId, + Nonce, + SubAssetId, + }, tai64, }; @@ -252,33 +267,267 @@ pub fn bytes32_to_vec(bytes: &fuel_core_types::fuel_types::Bytes32) -> Vec { bytes.as_ref().to_vec() } +fn script_execution_result_from_proto( + proto: &crate::protobuf_types::ScriptExecutionResult, +) -> crate::result::Result { + let variant = proto.variant.as_ref().ok_or_else(|| { + Error::Serialization(anyhow!("Missing script execution result variant")) + })?; + + let result = match variant { + ProtoScriptExecutionResultVariant::Success(_) => ScriptExecutionResult::Success, + ProtoScriptExecutionResultVariant::Revert(_) => ScriptExecutionResult::Revert, + ProtoScriptExecutionResultVariant::Panic(_) => ScriptExecutionResult::Panic, + ProtoScriptExecutionResultVariant::GenericFailure(failure) => { + ScriptExecutionResult::GenericFailure(failure.code) + } + }; + + Ok(result) +} + +fn panic_instruction_from_proto(proto: &ProtoPanicInstruction) -> PanicInstruction { + use crate::protobuf_types::PanicReason as ProtoPanicReason; + + let reason_proto = + ProtoPanicReason::try_from(proto.reason).unwrap_or(ProtoPanicReason::Unknown); + let reason = PanicReason::from(reason_proto as u8); + PanicInstruction::error(reason, proto.instruction) +} + +fn receipt_from_proto( + proto_receipt: &crate::protobuf_types::Receipt, +) -> crate::result::Result { + let variant = proto_receipt + .variant + .as_ref() + .ok_or_else(|| Error::Serialization(anyhow!("Missing receipt variant")))?; + + let receipt = match variant { + ProtoReceiptVariant::Call(call) => { + let id = ContractId::try_from(call.id.as_slice()) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + let to = ContractId::try_from(call.to.as_slice()) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + let asset_id = AssetId::try_from(call.asset_id.as_slice()) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + Ok(FuelReceipt::call( + id, + to, + call.amount, + asset_id, + call.gas, + call.param1, + call.param2, + call.pc, + call.is, + )) + } + ProtoReceiptVariant::ReturnReceipt(ret) => { + let id = ContractId::try_from(ret.id.as_slice()) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + Ok(FuelReceipt::ret(id, ret.val, ret.pc, ret.is)) + } + ProtoReceiptVariant::ReturnData(rd) => { + let id = ContractId::try_from(rd.id.as_slice()) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + let digest = Bytes32::try_from(rd.digest.as_slice()).map_err(|e| { + Error::Serialization(anyhow!( + "Could not convert return data digest to Bytes32: {}", + e + )) + })?; + Ok(FuelReceipt::return_data_with_len( + id, + rd.ptr, + rd.len, + digest, + rd.pc, + rd.is, + rd.data.clone(), + )) + } + ProtoReceiptVariant::Panic(panic_receipt) => { + let id = ContractId::try_from(panic_receipt.id.as_slice()) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + let reason_proto = panic_receipt + .reason + .as_ref() + .ok_or_else(|| Error::Serialization(anyhow!("Missing panic reason")))?; + let reason = panic_instruction_from_proto(reason_proto); + let contract_id = panic_receipt + .contract_id + .as_ref() + .map(|cid| { + ContractId::try_from(cid.as_slice()) + .map_err(|e| Error::Serialization(anyhow!(e))) + }) + .transpose()?; + Ok( + FuelReceipt::panic(id, reason, panic_receipt.pc, panic_receipt.is) + .with_panic_contract_id(contract_id), + ) + } + ProtoReceiptVariant::Revert(revert) => { + let id = ContractId::try_from(revert.id.as_slice()) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + Ok(FuelReceipt::revert(id, revert.ra, revert.pc, revert.is)) + } + ProtoReceiptVariant::Log(log) => { + let id = ContractId::try_from(log.id.as_slice()) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + Ok(FuelReceipt::log( + id, log.ra, log.rb, log.rc, log.rd, log.pc, log.is, + )) + } + ProtoReceiptVariant::LogData(log) => { + let id = ContractId::try_from(log.id.as_slice()) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + let digest = Bytes32::try_from(log.digest.as_slice()).map_err(|e| { + Error::Serialization(anyhow!( + "Could not convert log data digest to Bytes32: {}", + e + )) + })?; + Ok(FuelReceipt::log_data_with_len( + id, + log.ra, + log.rb, + log.ptr, + log.len, + digest, + log.pc, + log.is, + log.data.clone(), + )) + } + ProtoReceiptVariant::Transfer(transfer) => { + let id = ContractId::try_from(transfer.id.as_slice()) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + let to = ContractId::try_from(transfer.to.as_slice()) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + let asset_id = AssetId::try_from(transfer.asset_id.as_slice()) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + Ok(FuelReceipt::transfer( + id, + to, + transfer.amount, + asset_id, + transfer.pc, + transfer.is, + )) + } + ProtoReceiptVariant::TransferOut(transfer) => { + let id = ContractId::try_from(transfer.id.as_slice()) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + let to = Address::try_from(transfer.to.as_slice()) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + let asset_id = AssetId::try_from(transfer.asset_id.as_slice()) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + Ok(FuelReceipt::transfer_out( + id, + to, + transfer.amount, + asset_id, + transfer.pc, + transfer.is, + )) + } + ProtoReceiptVariant::ScriptResult(result) => { + let script_result = result.result.as_ref().ok_or_else(|| { + Error::Serialization(anyhow!("Missing script result payload")) + })?; + let execution_result = script_execution_result_from_proto(script_result)?; + Ok(FuelReceipt::script_result( + execution_result, + result.gas_used, + )) + } + ProtoReceiptVariant::MessageOut(msg) => { + let sender = Address::try_from(msg.sender.as_slice()) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + let recipient = Address::try_from(msg.recipient.as_slice()) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + let nonce = Nonce::try_from(msg.nonce.as_slice()) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + let digest = Bytes32::try_from(msg.digest.as_slice()).map_err(|e| { + Error::Serialization(anyhow!( + "Could not convert message digest to Bytes32: {}", + e + )) + })?; + Ok(FuelReceipt::message_out_with_len( + sender, + recipient, + msg.amount, + nonce, + msg.len, + digest, + msg.data.clone(), + )) + } + ProtoReceiptVariant::Mint(mint) => { + let sub_id = SubAssetId::try_from(mint.sub_id.as_slice()) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + let contract_id = ContractId::try_from(mint.contract_id.as_slice()) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + Ok(FuelReceipt::mint( + sub_id, + contract_id, + mint.val, + mint.pc, + mint.is, + )) + } + ProtoReceiptVariant::Burn(burn) => { + let sub_id = SubAssetId::try_from(burn.sub_id.as_slice()) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + let contract_id = ContractId::try_from(burn.contract_id.as_slice()) + .map_err(|e| Error::Serialization(anyhow!(e)))?; + Ok(FuelReceipt::burn( + sub_id, + contract_id, + burn.val, + burn.pc, + burn.is, + )) + } + }?; + + Ok(receipt) +} + pub fn fuel_block_from_protobuf( proto_block: ProtoBlock, msg_ids: &[fuel_core_types::fuel_tx::MessageId], event_inbox_root: Bytes32, -) -> crate::result::Result { +) -> crate::result::Result<(FuelBlock, Vec)> { let versioned_block = proto_block .versioned_block .ok_or_else(|| anyhow::anyhow!("Missing protobuf versioned_block")) .map_err(Error::Serialization)?; - let partial_header = match &versioned_block { - ProtoVersionedBlock::V1(v1_block) => { - let proto_header = v1_block + let (partial_header, txs, receipts) = match versioned_block { + ProtoVersionedBlock::V1(v1_inner) => { + let proto_header = v1_inner .header .clone() .ok_or_else(|| anyhow::anyhow!("Missing protobuf header")) .map_err(Error::Serialization)?; - partial_header_from_proto_header(&proto_header)? + let partial_header = partial_header_from_proto_header(&proto_header)?; + let txs = v1_inner + .transactions + .iter() + .map(tx_from_proto_tx) + .collect::>()?; + let receipts = v1_inner + .receipts + .iter() + .map(receipt_from_proto) + .collect::>()?; + (partial_header, txs, receipts) } }; - let txs = match versioned_block { - ProtoVersionedBlock::V1(v1_inner) => v1_inner - .transactions - .iter() - .map(tx_from_proto_tx) - .collect::>()?, - }; - FuelBlock::new( + let block = FuelBlock::new( partial_header, txs, msg_ids, @@ -287,7 +536,8 @@ pub fn fuel_block_from_protobuf( &ChainId::default(), ) .map_err(|e| anyhow!(e)) - .map_err(Error::Serialization) + .map_err(Error::Serialization)?; + Ok((block, receipts)) } pub fn partial_header_from_proto_header( diff --git a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/sync_service.rs b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/sync_service.rs index 59791bf0d61..15287180a6a 100644 --- a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/sync_service.rs +++ b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/sync_service.rs @@ -1,6 +1,12 @@ -use crate::blocks::{ - BlockSourceEvent, - importer_and_db_source::BlockSerializer, +use crate::{ + blocks::{ + BlockSourceEvent, + importer_and_db_source::BlockSerializer, + }, + result::{ + Error, + Result, + }, }; use fuel_core_services::{ RunnableService, @@ -12,6 +18,7 @@ use fuel_core_services::{ }; use fuel_core_storage::{ self, + Error as StorageError, StorageInspect, tables::{ FuelBlocks, @@ -19,66 +26,88 @@ use fuel_core_storage::{ }, }; use fuel_core_types::{ + blockchain::block::Block as FuelBlock, fuel_tx::{ + Receipt, Transaction, TxId, }, fuel_types::BlockHeight, }; +use futures::{ + StreamExt, + TryStreamExt, + stream::FuturesOrdered, +}; use tokio::sync::mpsc::Sender; -pub struct SyncTask { +pub struct SyncTask { serializer: Serializer, block_return_sender: Sender>, db: DB, + receipts: Receipts, next_height: BlockHeight, // exclusive, does not ask for this block stop_height: BlockHeight, } -impl SyncTask +pub trait TxReceipts: 'static + Send + Sync { + fn get_receipts( + &self, + tx_id: &TxId, + ) -> impl Future>> + Send; +} + +impl SyncTask where Serializer: BlockSerializer + Send, - DB: StorageInspect + Send + 'static, - DB: StorageInspect + Send + 'static, - E: std::fmt::Debug + Send, + DB: Send + Sync + 'static, + DB: StorageInspect, + DB: StorageInspect, + Receipts: TxReceipts, { pub fn new( serializer: Serializer, block_return: Sender>, db: DB, + receipts: Receipts, db_starting_height: BlockHeight, - // does not ask for this block + // does not ask for this block (exclusive) db_ending_height: BlockHeight, ) -> Self { Self { serializer, block_return_sender: block_return, db, + receipts, next_height: db_starting_height, stop_height: db_ending_height, } } - fn get_block( + async fn get_block_and_receipts( &self, height: &BlockHeight, - ) -> Result, E> { - let maybe_block = StorageInspect::::get(&self.db, height)?; + ) -> Result)>> { + let maybe_block = StorageInspect::::get(&self.db, height) + .map_err(Error::block_source_error)?; if let Some(block) = maybe_block { let tx_ids = block.transactions(); let txs = self.get_txs(tx_ids)?; + let receipts = self.get_receipts(tx_ids).await?; let block = block.into_owned().uncompress(txs); - Ok(Some(block)) + Ok(Some((block, receipts))) } else { Ok(None) } } - fn get_txs(&self, tx_ids: &[TxId]) -> Result, E> { + fn get_txs(&self, tx_ids: &[TxId]) -> Result> { let mut txs = Vec::new(); for tx_id in tx_ids { - match StorageInspect::::get(&self.db, tx_id)? { + match StorageInspect::::get(&self.db, tx_id) + .map_err(Error::block_source_error)? + { Some(tx) => { tracing::debug!("found tx id: {:?}", tx_id); txs.push(tx.into_owned()); @@ -90,16 +119,25 @@ where } Ok(txs) } + + async fn get_receipts(&self, tx_ids: &[TxId]) -> Result> { + let receipt_futs = tx_ids.iter().map(|tx_id| self.receipts.get_receipts(tx_id)); + FuturesOrdered::from_iter(receipt_futs) + .then(|res| async move { res.map_err(Error::block_source_error) }) + .try_concat() + .await + } } -impl RunnableTask for SyncTask +impl RunnableTask + for SyncTask where Serializer: BlockSerializer + Send + Sync, Serializer::Block: Send + Sync + 'static, DB: Send + Sync + 'static, - DB: StorageInspect + Send + 'static, - DB: StorageInspect + Send + 'static, - E: std::fmt::Debug + Send, + DB: StorageInspect, + DB: StorageInspect, + Receipts: TxReceipts, { async fn run(&mut self, _watcher: &mut StateWatcher) -> TaskNextAction { if self.next_height >= self.stop_height { @@ -110,16 +148,16 @@ where futures::future::pending().await } let next_height = self.next_height; - let res = self.get_block(&next_height); - let maybe_block = try_or_stop!(res, |e| { + let res = self.get_block_and_receipts(&next_height).await; + let maybe_block_and_receipts = try_or_stop!(res, |e| { tracing::error!("error fetching block at height {}: {:?}", next_height, e); }); - if let Some(block) = maybe_block { + if let Some((block, receipts)) = maybe_block_and_receipts { tracing::debug!( "found block at height {:?}, sending to return channel", next_height ); - let res = self.serializer.serialize_block(&block); + let res = self.serializer.serialize_block(&block, &receipts); let block = try_or_continue!(res); let event = BlockSourceEvent::OldBlock(BlockHeight::from(*next_height), block); @@ -139,14 +177,15 @@ where } #[async_trait::async_trait] -impl RunnableService for SyncTask +impl RunnableService + for SyncTask where Serializer: BlockSerializer + Send + Sync + 'static, ::Block: Send + Sync + 'static, DB: Send + Sync + 'static, - DB: StorageInspect + Send + 'static, - DB: StorageInspect + Send + 'static, - E: std::fmt::Debug + Send, + DB: StorageInspect, + DB: StorageInspect, + Receipts: TxReceipts, { const NAME: &'static str = "BlockSourceSyncTask"; type SharedData = (); diff --git a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/tests.rs b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/tests.rs index a04023ea537..9f2570d546e 100644 --- a/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/tests.rs +++ b/crates/services/block_aggregator_api/src/blocks/importer_and_db_source/tests.rs @@ -1,8 +1,6 @@ #![allow(non_snake_case)] use super::*; -use crate::blocks::BlockBytes; -use ::postcard::to_allocvec; use fuel_core_services::stream::{ IntoBoxStream, pending, @@ -18,11 +16,17 @@ use fuel_core_storage::{ }, }; use futures::StreamExt; +use std::collections::HashMap; +use crate::blocks::importer_and_db_source::{ + serializer_adapter::SerializerAdapter, + sync_service::TxReceipts, +}; use fuel_core_types::{ blockchain::SealedBlock, fuel_tx::{ Transaction, + TxId, UniqueIdentifier, }, fuel_types::ChainId, @@ -30,22 +34,28 @@ use fuel_core_types::{ }; use std::sync::Arc; -#[derive(Clone)] -pub struct MockSerializer; +fn onchain_db() -> StorageTransaction> { + InMemoryStorage::default().into_transaction() +} -impl BlockSerializer for MockSerializer { - type Block = BlockBytes; +struct MockTxReceiptsSource { + receipts_map: HashMap>, +} - fn serialize_block(&self, block: &FuelBlock) -> Result { - let bytes_vec = to_allocvec(block).map_err(|e| { - Error::BlockSource(anyhow!("failed to serialize block: {}", e)) - })?; - Ok(BlockBytes::from(bytes_vec)) +impl MockTxReceiptsSource { + fn new(receipts: &[(TxId, Vec)]) -> Self { + let receipts_map = receipts.iter().cloned().collect(); + Self { receipts_map } } } -fn database() -> StorageTransaction> { - InMemoryStorage::default().into_transaction() +impl TxReceipts for MockTxReceiptsSource { + async fn get_receipts(&self, tx_id: &TxId) -> Result> { + let receipts = self.receipts_map.get(tx_id).cloned().ok_or_else(|| { + Error::BlockSource(anyhow!("no receipts found for a tx with id {}", tx_id)) + })?; + Ok(receipts) + } } #[tokio::test] @@ -64,8 +74,9 @@ async fn next_block__gets_new_block_from_importer() { ); let blocks: Vec = vec![import_result]; let block_stream = tokio_stream::iter(blocks).chain(pending()).into_boxed(); - let serializer = MockSerializer; - let db = database(); + let serializer = SerializerAdapter; + let db = onchain_db(); + let receipt_source = MockTxReceiptsSource::new(&[]); let db_starting_height = BlockHeight::from(0u32); // we don't need to sync anything, so we can use the same height for both let db_ending_height = db_starting_height; @@ -73,6 +84,7 @@ async fn next_block__gets_new_block_from_importer() { block_stream, serializer.clone(), db, + receipt_source, db_starting_height, db_ending_height, ); @@ -81,7 +93,7 @@ async fn next_block__gets_new_block_from_importer() { let actual = adapter.next_block().await.unwrap(); // then - let serialized = serializer.serialize_block(&block.entity).unwrap(); + let serialized = serializer.serialize_block(&block.entity, &[]).unwrap(); let expected = BlockSourceEvent::NewBlock(*height, serialized); assert_eq!(expected, actual); } @@ -94,6 +106,25 @@ fn arbitrary_block_with_txs(height: BlockHeight) -> FuelBlock { block } +fn arbitrary_receipts() -> Vec { + let one = FuelReceipt::Mint { + sub_id: Default::default(), + contract_id: Default::default(), + val: 100, + pc: 0, + is: 0, + }; + let two = FuelReceipt::Transfer { + id: Default::default(), + to: Default::default(), + amount: 50, + asset_id: Default::default(), + pc: 0, + is: 0, + }; + vec![one, two] +} + #[tokio::test] async fn next_block__can_get_block_from_db() { // given @@ -101,14 +132,16 @@ async fn next_block__can_get_block_from_db() { let height1 = BlockHeight::from(0u32); let height2 = BlockHeight::from(1u32); let block = arbitrary_block_with_txs(height1); + let receipts = arbitrary_receipts(); let height = block.header().height(); - let serializer = MockSerializer; - let mut db = database(); - let mut tx = db.write_transaction(); + let serializer = SerializerAdapter; + let mut onchain_db = onchain_db(); + let mut tx = onchain_db.write_transaction(); let compressed_block = block.compress(&chain_id); tx.storage_as_mut::() .insert(height, &compressed_block) .unwrap(); + let tx_id = block.transactions()[0].id(&chain_id); tx.storage_as_mut::() .insert( &block.transactions()[0].id(&chain_id), @@ -116,13 +149,15 @@ async fn next_block__can_get_block_from_db() { ) .unwrap(); tx.commit().unwrap(); + let receipt_source = MockTxReceiptsSource::new(&[(tx_id, receipts.clone())]); let block_stream = tokio_stream::pending().into_boxed(); let db_starting_height = *height; let db_ending_height = height2; let mut adapter = ImporterAndDbSource::new( block_stream, serializer.clone(), - db, + onchain_db, + receipt_source, db_starting_height, db_ending_height, ); @@ -131,7 +166,7 @@ async fn next_block__can_get_block_from_db() { let actual = adapter.next_block().await.unwrap(); // then - let serialized = serializer.serialize_block(&block).unwrap(); + let serialized = serializer.serialize_block(&block, &receipts).unwrap(); let expected = BlockSourceEvent::OldBlock(*height, serialized); assert_eq!(expected, actual); } diff --git a/crates/services/block_aggregator_api/src/db/remote_cache.rs b/crates/services/block_aggregator_api/src/db/remote_cache.rs index 884deff70d3..db3a385a008 100644 --- a/crates/services/block_aggregator_api/src/db/remote_cache.rs +++ b/crates/services/block_aggregator_api/src/db/remote_cache.rs @@ -9,11 +9,19 @@ use crate::{ result::Error, }; use anyhow::anyhow; +use aws_config::{ + BehaviorVersion, + default_provider::credentials::DefaultCredentialsChain, +}; use aws_sdk_s3::{ self, Client, primitives::ByteStream, }; +use flate2::{ + Compression, + write::GzEncoder, +}; use fuel_core_storage::{ Error as StorageError, StorageAsMut, @@ -28,6 +36,7 @@ use fuel_core_storage::{ }; use fuel_core_types::fuel_types::BlockHeight; use prost::Message; +use std::io::Write; #[allow(non_snake_case)] #[cfg(test)] @@ -36,12 +45,10 @@ mod tests; #[allow(unused)] pub struct RemoteCache { // aws configuration - aws_id: String, - aws_secret: String, - aws_region: String, aws_bucket: String, - url_base: String, - client: Client, + requester_pays: bool, + aws_endpoint: Option, + client: Option, // track consistency between runs local_persisted: S, @@ -54,21 +61,17 @@ pub struct RemoteCache { impl RemoteCache { #[allow(clippy::too_many_arguments)] pub fn new( - aws_id: String, - aws_secret: String, - aws_region: String, aws_bucket: String, - url_base: String, - client: Client, + requester_pays: bool, + aws_endpoint: Option, + client: Option, local_persisted: S, sync_from: BlockHeight, ) -> RemoteCache { RemoteCache { - aws_id, - aws_secret, - aws_region, aws_bucket, - url_base, + requester_pays, + aws_endpoint, client, local_persisted, sync_from, @@ -78,8 +81,29 @@ impl RemoteCache { } } - fn url_for_block(base: &str, key: &str) -> String { - format!("{}/{}", base, key,) + async fn client(&mut self) -> crate::result::Result<&Client> { + self.init_client().await; + self.client + .as_ref() + .ok_or(Error::db_error(anyhow!("AWS S3 client is uninitialized"))) + } + + // only runs the first time + async fn init_client(&mut self) { + if self.client.is_none() { + let credentials = DefaultCredentialsChain::builder().build().await; + let sdk_config = aws_config::defaults(BehaviorVersion::latest()) + .credentials_provider(credentials) + .load() + .await; + let mut config_builder = aws_sdk_s3::config::Builder::from(&sdk_config); + if let Some(endpoint) = &self.aws_endpoint { + config_builder.set_endpoint_url(Some(endpoint.to_string())); + } + let config = config_builder.force_path_style(true).build(); + let client = aws_sdk_s3::Client::from_conf(config); + self.client = Some(client); + } } } @@ -102,13 +126,16 @@ where let key = block_height_to_key(&height); let mut buf = Vec::new(); block.encode(&mut buf).map_err(Error::db_error)?; - let body = ByteStream::from(buf); + let zipped = gzip_bytes(&buf)?; + let body = ByteStream::from(zipped); let req = self - .client + .client() + .await? .put_object() .bucket(&self.aws_bucket) .key(&key) .body(body) + .content_encoding("gzip") .content_type("application/octet-stream"); let _ = req.send().await.map_err(Error::db_error)?; match block_event { @@ -176,21 +203,22 @@ where last: BlockHeight, ) -> crate::result::Result { // TODO: Check if it exists - let region = self.aws_region.clone(); let bucket = self.aws_bucket.clone(); - let base = self.url_base.clone(); + let requester_pays = self.requester_pays; + let aws_endpoint = self.aws_endpoint.clone(); let stream = futures::stream::iter((*first..=*last).map(move |height| { - let key = block_height_to_key(&BlockHeight::new(height)); - let url = Self::url_for_block(&base, &key); - crate::block_range_response::RemoteBlockRangeResponse { - region: region.clone(), + let block_height = BlockHeight::new(height); + let key = block_height_to_key(&block_height); + let res = crate::block_range_response::RemoteS3Response { bucket: bucket.clone(), key: key.clone(), - url, - } + requester_pays, + aws_endpoint: aws_endpoint.clone(), + }; + (block_height, res) })); - Ok(BlockRangeResponse::Remote(Box::pin(stream))) + Ok(BlockRangeResponse::S3(Box::pin(stream))) } async fn get_current_height(&self) -> crate::result::Result> { @@ -227,5 +255,15 @@ where } pub fn block_height_to_key(height: &BlockHeight) -> String { - format!("{:08x}", height) + let raw: [u8; 4] = height.to_bytes(); + format!( + "{:02}/{:02}/{:02}/{:02}", + &raw[0], &raw[1], &raw[2], &raw[3] + ) +} + +pub fn gzip_bytes(data: &[u8]) -> crate::result::Result> { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(data).map_err(Error::db_error)?; + encoder.finish().map_err(Error::db_error) } diff --git a/crates/services/block_aggregator_api/src/db/remote_cache/tests.rs b/crates/services/block_aggregator_api/src/db/remote_cache/tests.rs index 16b52776de7..1e261281953 100644 --- a/crates/services/block_aggregator_api/src/db/remote_cache/tests.rs +++ b/crates/services/block_aggregator_api/src/db/remote_cache/tests.rs @@ -1,6 +1,6 @@ use super::*; use crate::{ - block_range_response::RemoteBlockRangeResponse, + block_range_response::RemoteS3Response, blocks::importer_and_db_source::{ BlockSerializer, serializer_adapter::SerializerAdapter, @@ -31,7 +31,7 @@ fn database() -> StorageTransaction> { fn arb_proto_block() -> ProtoBlock { let block = FuelBlock::default(); let serializer = SerializerAdapter; - serializer.serialize_block(&block).unwrap() + serializer.serialize_block(&block, &[]).unwrap() } fn put_happy_rule() -> Rule { mock!(Client::put_object) @@ -45,16 +45,11 @@ fn put_happy_rule() -> Rule { async fn store_block__happy_path() { // given let client = mock_client!(aws_sdk_s3, [&put_happy_rule()]); - let aws_id = "test-id".to_string(); - let aws_secret = "test-secret".to_string(); - let aws_region = "test-region".to_string(); let aws_bucket = "test-bucket".to_string(); - let base = "http://good.com".to_string(); let storage = database(); let sync_from = BlockHeight::new(0); - let mut adapter = RemoteCache::new( - aws_id, aws_secret, aws_region, aws_bucket, base, client, storage, sync_from, - ); + let mut adapter = + RemoteCache::new(aws_bucket, false, None, Some(client), storage, sync_from); let block_height = BlockHeight::new(123); let block = arb_proto_block(); let block = BlockSourceEvent::OldBlock(block_height, block); @@ -70,20 +65,14 @@ async fn store_block__happy_path() { async fn get_block_range__happy_path() { // given let client = mock_client!(aws_sdk_s3, []); - let aws_id = "test-id".to_string(); - let aws_secret = "test-secret".to_string(); - let aws_region = "test-region".to_string(); let aws_bucket = "test-bucket".to_string(); - let base = "http://good.com".to_string(); let storage = database(); let sync_from = BlockHeight::new(0); let adapter = RemoteCache::new( - aws_id, - aws_secret, - aws_region.clone(), aws_bucket.clone(), - base.clone(), - client, + false, + None, + Some(client), storage, sync_from, ); @@ -98,18 +87,18 @@ async fn get_block_range__happy_path() { BlockRangeResponse::Literal(_) => { panic!("Expected remote response, got literal"); } - BlockRangeResponse::Remote(stream) => stream.collect::>().await, + BlockRangeResponse::S3(stream) => stream.collect::>().await, }; let expected = (999..=1003) .map(|height| { let key = block_height_to_key(&BlockHeight::new(height)); - let url = RemoteCache::<()>::url_for_block(&base, &key); - RemoteBlockRangeResponse { - region: aws_region.clone(), + let res = RemoteS3Response { bucket: aws_bucket.clone(), key, - url, - } + requester_pays: false, + aws_endpoint: None, + }; + (BlockHeight::new(height), res) }) .collect::>(); assert_eq!(actual, expected); @@ -119,16 +108,11 @@ async fn get_block_range__happy_path() { async fn get_current_height__returns_highest_continuous_block() { // given let client = mock_client!(aws_sdk_s3, [&put_happy_rule()]); - let aws_id = "test-id".to_string(); - let aws_secret = "test-secret".to_string(); - let aws_region = "test-region".to_string(); let aws_bucket = "test-bucket".to_string(); - let base = "http://good.com".to_string(); let storage = database(); let sync_from = BlockHeight::new(0); - let mut adapter = RemoteCache::new( - aws_id, aws_secret, aws_region, aws_bucket, base, client, storage, sync_from, - ); + let mut adapter = + RemoteCache::new(aws_bucket, false, None, Some(client), storage, sync_from); let expected = BlockHeight::new(123); let block = arb_proto_block(); @@ -153,15 +137,10 @@ async fn store_block__does_not_update_the_highest_continuous_block_if_not_contig .unwrap(); tx.commit().unwrap(); let client = mock_client!(aws_sdk_s3, [&put_happy_rule()]); - let aws_id = "test-id".to_string(); - let aws_secret = "test-secret".to_string(); - let aws_region = "test-region".to_string(); let aws_bucket = "test-bucket".to_string(); - let base = "http://good.com".to_string(); let sync_from = BlockHeight::new(0); - let mut adapter = RemoteCache::new( - aws_id, aws_secret, aws_region, aws_bucket, base, client, storage, sync_from, - ); + let mut adapter = + RemoteCache::new(aws_bucket, false, None, Some(client), storage, sync_from); let expected = BlockHeight::new(3); let block = arb_proto_block(); @@ -178,18 +157,13 @@ async fn store_block__does_not_update_the_highest_continuous_block_if_not_contig async fn store_block__updates_the_highest_continuous_block_if_filling_a_gap() { let rules: Vec<_> = iter::repeat_with(put_happy_rule).take(10).collect(); let client = mock_client!(aws_sdk_s3, rules.iter()); - let aws_id = "test-id".to_string(); - let aws_secret = "test-secret".to_string(); - let aws_region = "test-region".to_string(); let aws_bucket = "test-bucket".to_string(); - let base = "http://good.com".to_string(); // given let db = database(); let sync_from = BlockHeight::new(0); - let mut adapter = RemoteCache::new( - aws_id, aws_secret, aws_region, aws_bucket, base, client, db, sync_from, - ); + let mut adapter = + RemoteCache::new(aws_bucket, false, None, Some(client), db, sync_from); for height in 2..=10u32 { let height = BlockHeight::from(height); @@ -215,18 +189,13 @@ async fn store_block__updates_the_highest_continuous_block_if_filling_a_gap() { async fn store_block__new_block_updates_the_highest_continuous_block_if_synced() { let rules: Vec<_> = iter::repeat_with(put_happy_rule).take(10).collect(); let client = mock_client!(aws_sdk_s3, rules.iter()); - let aws_id = "test-id".to_string(); - let aws_secret = "test-secret".to_string(); - let aws_region = "test-region".to_string(); let aws_bucket = "test-bucket".to_string(); - let base = "http://good.com".to_string(); // given let db = database(); let sync_from = BlockHeight::new(0); - let mut adapter = RemoteCache::new( - aws_id, aws_secret, aws_region, aws_bucket, base, client, db, sync_from, - ); + let mut adapter = + RemoteCache::new(aws_bucket, false, None, Some(client), db, sync_from); let height = BlockHeight::from(0u32); let some_block = arb_proto_block(); @@ -251,18 +220,13 @@ async fn store_block__new_block_updates_the_highest_continuous_block_if_synced() async fn store_block__new_block_comes_first() { let rules: Vec<_> = iter::repeat_with(put_happy_rule).take(10).collect(); let client = mock_client!(aws_sdk_s3, rules.iter()); - let aws_id = "test-id".to_string(); - let aws_secret = "test-secret".to_string(); - let aws_region = "test-region".to_string(); let aws_bucket = "test-bucket".to_string(); - let base = "http://good.com".to_string(); // given let db = database(); let sync_from = BlockHeight::new(0); - let mut adapter = RemoteCache::new( - aws_id, aws_secret, aws_region, aws_bucket, base, client, db, sync_from, - ); + let mut adapter = + RemoteCache::new(aws_bucket, false, None, Some(client), db, sync_from); // when let height = BlockHeight::from(0u32); diff --git a/crates/services/block_aggregator_api/src/db/storage_db.rs b/crates/services/block_aggregator_api/src/db/storage_db.rs index 4672e420625..d3f701748c2 100644 --- a/crates/services/block_aggregator_api/src/db/storage_db.rs +++ b/crates/services/block_aggregator_api/src/db/storage_db.rs @@ -207,7 +207,7 @@ where S: Unpin + ReadTransaction + std::fmt::Debug, for<'a> StorageTransaction<&'a S>: StorageInspect, { - type Item = ProtoBlock; + type Item = (BlockHeight, ProtoBlock); fn poll_next( self: Pin<&mut Self>, @@ -233,7 +233,7 @@ where None }; this.next = next; - Poll::Ready(Some(block.into_owned())) + Poll::Ready(Some((height, block.into_owned()))) } Ok(None) => { tracing::debug!("No block at height: {:?}", height); diff --git a/crates/services/block_aggregator_api/src/db/storage_db/tests.rs b/crates/services/block_aggregator_api/src/db/storage_db/tests.rs index fe030080da0..6ed9f2c851c 100644 --- a/crates/services/block_aggregator_api/src/db/storage_db/tests.rs +++ b/crates/services/block_aggregator_api/src/db/storage_db/tests.rs @@ -28,7 +28,9 @@ fn proto_block_with_height(height: BlockHeight) -> ProtoBlock { let serializer_adapter = SerializerAdapter; let mut default_block = FuelBlock::::default(); default_block.header_mut().set_block_height(height); - serializer_adapter.serialize_block(&default_block).unwrap() + serializer_adapter + .serialize_block(&default_block, &[]) + .unwrap() } #[tokio::test] @@ -90,7 +92,7 @@ async fn get_block__can_get_expected_range() { let actual = stream.collect::>().await; // then - assert_eq!(actual, vec![expected_2, expected_3]); + assert_eq!(actual, vec![(height_2, expected_2), (height_3, expected_3)]); } #[tokio::test] diff --git a/crates/services/block_aggregator_api/src/db/storage_or_remote_db.rs b/crates/services/block_aggregator_api/src/db/storage_or_remote_db.rs index fe14512cfcf..1236ccda961 100644 --- a/crates/services/block_aggregator_api/src/db/storage_or_remote_db.rs +++ b/crates/services/block_aggregator_api/src/db/storage_or_remote_db.rs @@ -13,10 +13,7 @@ use crate::{ }, result::Result, }; -use aws_sdk_s3::config::{ - Credentials, - Region, -}; + use fuel_core_storage::{ Error as StorageError, StorageInspect, @@ -29,7 +26,6 @@ use fuel_core_storage::{ }, }; use fuel_core_types::fuel_types::BlockHeight; -use std::borrow::Cow; /// A union of a storage and a remote cache for the block aggregator. This allows both to be /// supported in production depending on the configuration @@ -46,41 +42,16 @@ impl StorageOrRemoteDB { #[allow(clippy::too_many_arguments)] pub fn new_s3( storage: R, - aws_id: &str, - aws_secret: &str, - aws_region: &str, aws_bucket: &str, - url_base: &str, + requester_pays: bool, aws_endpoint_url: Option, sync_from: BlockHeight, ) -> Self { - let client = { - let mut builder = aws_sdk_s3::config::Builder::new(); - if let Some(aws_endpoint_url) = aws_endpoint_url { - builder.set_endpoint_url(Some(aws_endpoint_url.clone())); - } - - let config = builder - .force_path_style(true) - .region(Region::new(Cow::Owned(aws_region.to_string()))) - .credentials_provider(Credentials::new( - aws_id, - aws_secret, - None, - None, - "block-aggregator", - )) - .behavior_version_latest() - .build(); - aws_sdk_s3::Client::from_conf(config) - }; let remote_cache = RemoteCache::new( - aws_id.to_string(), - aws_secret.to_string(), - aws_region.to_string(), aws_bucket.to_string(), - url_base.to_string(), - client, + requester_pays, + aws_endpoint_url, + None, storage, sync_from, ); @@ -88,24 +59,6 @@ impl StorageOrRemoteDB { } } -pub fn get_env_vars() -> Option<(String, String, String, String, String, Option)> -{ - let aws_id = std::env::var("AWS_ACCESS_KEY_ID").ok()?; - let aws_secret = std::env::var("AWS_SECRET_ACCESS_KEY").ok()?; - let aws_region = std::env::var("AWS_REGION").ok()?; - let aws_bucket = std::env::var("AWS_BUCKET").ok()?; - let bucket_url_base = std::env::var("BUCKET_URL_BASE").ok()?; - let aws_endpoint_url = std::env::var("AWS_ENDPOINT_URL").ok(); - Some(( - aws_id, - aws_secret, - aws_region, - aws_bucket, - bucket_url_base, - aws_endpoint_url, - )) -} - impl BlockAggregatorDB for StorageOrRemoteDB where // Storage Constraints diff --git a/crates/services/block_aggregator_api/src/lib.rs b/crates/services/block_aggregator_api/src/lib.rs index 90572e2cf24..ffdeb220800 100644 --- a/crates/services/block_aggregator_api/src/lib.rs +++ b/crates/services/block_aggregator_api/src/lib.rs @@ -36,6 +36,7 @@ pub mod integration { blocks::importer_and_db_source::{ BlockSerializer, ImporterAndDbSource, + sync_service::TxReceipts, }, db::BlockAggregatorDB, protobuf_types::Block as ProtoBlock, @@ -45,6 +46,7 @@ pub mod integration { stream::BoxStream, }; use fuel_core_storage::{ + Error as StorageError, StorageInspect, tables::{ FuelBlocks, @@ -62,21 +64,34 @@ pub mod integration { pub struct Config { pub addr: SocketAddr, pub sync_from: Option, + pub storage_method: StorageMethod, + } + + #[derive(Clone, Debug, Default)] + pub enum StorageMethod { + #[default] + Local, + S3 { + bucket: String, + endpoint_url: Option, + requester_pays: bool, + }, } #[allow(clippy::type_complexity)] - pub fn new_service( + pub fn new_service( config: &Config, db: DB, serializer: S, onchain_db: OnchainDB, + receipts: Receipts, importer: BoxStream, sync_from_height: BlockHeight, ) -> anyhow::Result, + ImporterAndDbSource, ProtoBlock, >, >> @@ -87,10 +102,10 @@ pub mod integration { >, S: BlockSerializer + Clone + Send + Sync + 'static, OnchainDB: Send + Sync, - OnchainDB: StorageInspect, - OnchainDB: StorageInspect, + OnchainDB: StorageInspect, + OnchainDB: StorageInspect, OnchainDB: HistoricalView, - E: std::fmt::Debug + Send + Sync, + Receipts: TxReceipts, { let addr = config.addr.to_string(); let api = ProtobufAPI::new(addr) @@ -103,6 +118,7 @@ pub mod integration { importer, serializer, onchain_db, + receipts, sync_from_height, db_ending_height, ); @@ -125,7 +141,7 @@ pub struct BlockAggregator { query: Api, database: DB, block_source: Blocks, - new_block_subscriptions: Vec>, + new_block_subscriptions: Vec>, } pub struct NewBlock { diff --git a/crates/services/block_aggregator_api/src/result.rs b/crates/services/block_aggregator_api/src/result.rs index bbe500cab56..3a49b0b58ff 100644 --- a/crates/services/block_aggregator_api/src/result.rs +++ b/crates/services/block_aggregator_api/src/result.rs @@ -9,12 +9,22 @@ pub enum Error { DB(anyhow::Error), #[error("Serialization error: {0}")] Serialization(anyhow::Error), + #[error("Receipt error: {0}")] + Receipt(anyhow::Error), } impl Error { pub fn db_error>(err: T) -> Self { Error::DB(err.into()) } + + pub fn block_source_error>(err: T) -> Self { + Error::BlockSource(err.into()) + } + + pub fn receipt_error>(err: T) -> Self { + Error::Receipt(err.into()) + } } pub type Result = core::result::Result; diff --git a/crates/services/block_aggregator_api/src/tests.rs b/crates/services/block_aggregator_api/src/tests.rs index 81feb61d064..dc00e5a0efe 100644 --- a/crates/services/block_aggregator_api/src/tests.rs +++ b/crates/services/block_aggregator_api/src/tests.rs @@ -249,7 +249,7 @@ async fn run__new_block_subscription__sends_new_block() { // then let actual_block = await_response_with_timeout(response).await.unwrap(); - assert_eq!(expected_block, actual_block); + assert_eq!((expected_height, expected_block), actual_block); // cleanup drop(source_sender); diff --git a/crates/services/consensus_module/poa/src/service.rs b/crates/services/consensus_module/poa/src/service.rs index 6545f32f653..2867d64688d 100644 --- a/crates/services/consensus_module/poa/src/service.rs +++ b/crates/services/consensus_module/poa/src/service.rs @@ -523,7 +523,7 @@ where return match res { Ok(()) => Some(TaskNextAction::Continue), Err(err) => Some(TaskNextAction::ErrorContinue(err)), - } + }; } None } @@ -599,7 +599,7 @@ where return Ok(Self { last_block_created: Instant::now(), ..self - }) + }); } } diff --git a/crates/services/executor/src/executor.rs b/crates/services/executor/src/executor.rs index 9a0591f5709..8906f742c0f 100644 --- a/crates/services/executor/src/executor.rs +++ b/crates/services/executor/src/executor.rs @@ -1670,12 +1670,12 @@ where let Input::Contract(input) = core::mem::take(input) else { return Err(ExecutorError::Other( "Input of the `Mint` transaction is not a contract".to_string(), - )) + )); }; let Output::Contract(output) = outputs[0] else { return Err(ExecutorError::Other( "The output of the `Mint` transaction is not a contract".to_string(), - )) + )); }; Ok((input, output)) } @@ -1793,7 +1793,7 @@ where ); return Err(ExecutorError::InvalidTransactionOutcome { transaction_id: tx_id, - }) + }); } } } @@ -2026,14 +2026,14 @@ where return Err(TransactionValidityError::CoinMismatch( *utxo_id, ) - .into()) + .into()); } } _ => { return Err(TransactionValidityError::CoinDoesNotExist( *utxo_id, ) - .into()) + .into()); } } } @@ -2045,7 +2045,7 @@ where return Err(TransactionValidityError::ContractDoesNotExist( contract.contract_id, ) - .into()) + .into()); } } Input::MessageCoinSigned(MessageCoinSigned { nonce, .. }) @@ -2060,21 +2060,21 @@ where *nonce, ) .into(), - ) + ); } if !message.matches_input(input).unwrap_or_default() { return Err(TransactionValidityError::MessageMismatch( *nonce, ) - .into()) + .into()); } } _ => { return Err(TransactionValidityError::MessageDoesNotExist( *nonce, ) - .into()) + .into()); } } } @@ -2133,7 +2133,7 @@ where if reverted => { // Don't spend the retryable messages if transaction is reverted - continue + continue; } Input::MessageCoinSigned(MessageCoinSigned { nonce, .. }) | Input::MessageCoinPredicate(MessageCoinPredicate { nonce, .. }) @@ -2170,7 +2170,7 @@ where for r in receipts.iter().rev() { if let Receipt::ScriptResult { gas_used, .. } = r { used_gas = *gas_used; - break + break; } } @@ -2278,7 +2278,7 @@ where } else { return Err(ExecutorError::InvalidTransactionOutcome { transaction_id: tx_id, - }) + }); }; let empty = ContractAccessesWithValues::default(); @@ -2369,7 +2369,7 @@ where } else { return Err(ExecutorError::TransactionValidity( TransactionValidityError::InvalidContractInputIndex(utxo_id), - )) + )); } } Output::Change { diff --git a/crates/services/p2p/src/peer_manager.rs b/crates/services/p2p/src/peer_manager.rs index 92a0c3ab9b8..fcdb1ab69d3 100644 --- a/crates/services/p2p/src/peer_manager.rs +++ b/crates/services/p2p/src/peer_manager.rs @@ -252,7 +252,7 @@ impl PeerManager { // check if all the slots are already taken if non_reserved_peers_connected >= self.max_non_reserved_peers { // Too many peers already connected, disconnect the Peer - return true + return true; } if non_reserved_peers_connected.saturating_add(1) diff --git a/crates/services/producer/src/block_producer.rs b/crates/services/producer/src/block_producer.rs index a7d09072839..2505fc84d02 100644 --- a/crates/services/producer/src/block_producer.rs +++ b/crates/services/producer/src/block_producer.rs @@ -145,7 +145,7 @@ where height, previous_block: latest_height, } - .into()) + .into()); } let maybe_mint_tx = transactions_source.pop(); @@ -229,7 +229,7 @@ where height, previous_block: latest_height, } - .into()) + .into()); } let component = Components { diff --git a/crates/services/shared-sequencer/src/service.rs b/crates/services/shared-sequencer/src/service.rs index c81bf1b63be..b807b0a9229 100644 --- a/crates/services/shared-sequencer/src/service.rs +++ b/crates/services/shared-sequencer/src/service.rs @@ -178,7 +178,7 @@ where async fn run(&mut self, watcher: &mut StateWatcher) -> TaskNextAction { if !self.config.enabled { let _ = watcher.while_started().await; - return TaskNextAction::Stop; + return TaskNextAction::Stop } if let Err(err) = self.ensure_account_metadata().await { diff --git a/crates/services/txpool_v2/src/selection_algorithms/ratio_tip_gas.rs b/crates/services/txpool_v2/src/selection_algorithms/ratio_tip_gas.rs index e944959f3f3..05465a0f751 100644 --- a/crates/services/txpool_v2/src/selection_algorithms/ratio_tip_gas.rs +++ b/crates/services/txpool_v2/src/selection_algorithms/ratio_tip_gas.rs @@ -206,7 +206,7 @@ where < constraints.minimal_gas_price; if less_price { - continue; + continue } let not_enough_gas = stored_transaction.transaction.max_gas() > gas_left; @@ -214,7 +214,7 @@ where stored_transaction.transaction.metered_bytes_size() > space_left; if not_enough_gas || too_big_tx { - continue; + continue } gas_left = diff --git a/crates/services/txpool_v2/src/service.rs b/crates/services/txpool_v2/src/service.rs index 8f6af4fb3e4..9db5ac74b1c 100644 --- a/crates/services/txpool_v2/src/service.rs +++ b/crates/services/txpool_v2/src/service.rs @@ -581,7 +581,7 @@ where // We already synced with this peer in the past. if !tx_sync_history.insert(peer_id.clone()) { - return + return; } } diff --git a/crates/services/txpool_v2/src/storage/graph.rs b/crates/services/txpool_v2/src/storage/graph.rs index d042f445d0d..1a0a741ef28 100644 --- a/crates/services/txpool_v2/src/storage/graph.rs +++ b/crates/services/txpool_v2/src/storage/graph.rs @@ -221,17 +221,17 @@ impl GraphStorage { if to != i_owner { return Err(Error::InputValidation( InputValidationError::NotInsertedIoWrongOwner, - )); + )) } if amount != i_amount { return Err(Error::InputValidation( InputValidationError::NotInsertedIoWrongAmount, - )); + )) } if asset_id != i_asset_id { return Err(Error::InputValidation( InputValidationError::NotInsertedIoWrongAssetId, - )); + )) } } Output::Contract(_) => { @@ -687,10 +687,10 @@ impl Storage for GraphStorage { if extracted_outputs .coin_exists(utxo_id, owner, amount, asset_id) { - continue; + continue } missing_inputs.push(MissingInput::Utxo(*utxo_id)); - continue; + continue } Err(e) => { return Err(InputValidationErrorType::Inconsistency( @@ -746,10 +746,10 @@ impl Storage for GraphStorage { Ok(true) => {} Ok(false) => { if extracted_outputs.contract_exists(contract_id) { - continue; + continue } missing_inputs.push(MissingInput::Contract(*contract_id)); - continue; + continue } Err(e) => { return Err(InputValidationErrorType::Inconsistency( diff --git a/crates/types/src/test_helpers.rs b/crates/types/src/test_helpers.rs index 1dbc750135d..014d8d4b563 100644 --- a/crates/types/src/test_helpers.rs +++ b/crates/types/src/test_helpers.rs @@ -12,6 +12,7 @@ use crate::{ v1::GeneratedApplicationFieldsV1, }, }, + fuel_asm::PanicInstruction, fuel_merkle::binary::root_calculator::MerkleRootCalculator, fuel_tx::{ BlobBody, @@ -23,6 +24,8 @@ use crate::{ Input, MessageId, Output, + Receipt, + ScriptExecutionResult, StorageSlot, Transaction, TransactionBuilder, @@ -43,6 +46,7 @@ use crate::{ BlobId, BlockHeight, Nonce, + SubAssetId, }, fuel_vm::{ Contract, @@ -430,6 +434,27 @@ prop_compose! { } } +fn arb_contract_id() -> impl Strategy { + any::<[u8; 32]>().prop_map(ContractId::new) +} + +fn arb_sub_asset_id() -> impl Strategy { + any::<[u8; 32]>().prop_map(SubAssetId::new) +} + +fn arb_panic_instruction() -> impl Strategy { + any::().prop_map(PanicInstruction::from) +} + +fn arb_script_execution_result() -> impl Strategy { + prop_oneof![ + Just(ScriptExecutionResult::Success), + Just(ScriptExecutionResult::Revert), + Just(ScriptExecutionResult::Panic), + any::().prop_map(ScriptExecutionResult::GenericFailure), + ] +} + fn arb_msg_ids() -> impl Strategy> { prop::collection::vec(arb_msg_id(), 0..10usize) } @@ -670,3 +695,145 @@ prop_compose! { (fuel_block, msg_ids, event_root) } } + +fn arb_receipt() -> impl Strategy { + prop_oneof![ + ( + arb_contract_id(), + arb_contract_id(), + any::(), + arb_asset_id(), + any::(), + any::(), + any::(), + any::(), + any::(), + ) + .prop_map( + |(id, to, amount, asset_id, gas, param1, param2, pc, is)| { + Receipt::call(id, to, amount, asset_id, gas, param1, param2, pc, is) + }, + ), + (arb_contract_id(), any::(), any::(), any::(),) + .prop_map(|(id, val, pc, is)| Receipt::ret(id, val, pc, is)), + ( + arb_contract_id(), + any::(), + any::(), + any::(), + prop::collection::vec(any::(), 0..64), + ) + .prop_map(|(id, ptr, pc, is, data)| Receipt::return_data( + id, ptr, pc, is, data, + )), + ( + arb_contract_id(), + arb_panic_instruction(), + any::(), + any::(), + prop::option::of(arb_contract_id()), + ) + .prop_map(|(id, reason, pc, is, panic_contract)| { + Receipt::panic(id, reason, pc, is).with_panic_contract_id(panic_contract) + }), + (arb_contract_id(), any::(), any::(), any::(),) + .prop_map(|(id, ra, pc, is)| Receipt::revert(id, ra, pc, is)), + ( + arb_contract_id(), + any::(), + any::(), + any::(), + any::(), + any::(), + any::(), + ) + .prop_map(|(id, ra, rb, rc, rd, pc, is)| { + Receipt::log(id, ra, rb, rc, rd, pc, is) + }), + ( + arb_contract_id(), + any::(), + any::(), + any::(), + any::(), + any::(), + prop::collection::vec(any::(), 0..64), + ) + .prop_map(|(id, ra, rb, ptr, pc, is, data)| { + Receipt::log_data(id, ra, rb, ptr, pc, is, data) + }), + ( + arb_contract_id(), + arb_contract_id(), + any::(), + arb_asset_id(), + any::(), + any::(), + ) + .prop_map(|(id, to, amount, asset_id, pc, is)| { + Receipt::transfer(id, to, amount, asset_id, pc, is) + }), + ( + arb_contract_id(), + arb_address(), + any::(), + arb_asset_id(), + any::(), + any::(), + ) + .prop_map(|(id, to, amount, asset_id, pc, is)| { + Receipt::transfer_out(id, to, amount, asset_id, pc, is) + }), + (arb_script_execution_result(), any::()) + .prop_map(|(result, gas_used)| Receipt::script_result(result, gas_used),), + ( + arb_address(), + arb_address(), + any::(), + arb_nonce(), + prop::collection::vec(any::(), 0..64), + ) + .prop_map(|(sender, recipient, amount, nonce, data)| { + let len = data.len() as u64; + let digest = Output::message_digest(&data); + Receipt::message_out_with_len( + sender, + recipient, + amount, + nonce, + len, + digest, + Some(data), + ) + }), + ( + arb_sub_asset_id(), + arb_contract_id(), + any::(), + any::(), + any::(), + ) + .prop_map(|(sub_id, contract_id, val, pc, is)| { + Receipt::mint(sub_id, contract_id, val, pc, is) + }), + ( + arb_sub_asset_id(), + arb_contract_id(), + any::(), + any::(), + any::(), + ) + .prop_map(|(sub_id, contract_id, val, pc, is)| { + Receipt::burn(sub_id, contract_id, val, pc, is) + }), + ] +} + +prop_compose! { + /// generates a list of random receipts + pub fn arb_receipts()( + receipts in prop::collection::vec(arb_receipt(), 0..10), + ) -> Vec { + receipts + } +} diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 27b6472c395..6e5f896ccba 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -101,6 +101,8 @@ url = { workspace = true } alloy-primitives = { workspace = true } alloy-provider = { workspace = true, default-features = false, features = ["reqwest-rustls-tls"] } alloy-rpc-types-eth = { workspace = true } +aws-config = "1.8.10" +flate2 = { workspace = true } fuel-core-executor = { workspace = true, features = ["limited-tx-count"] } pretty_assertions = "1.4" proptest = { workspace = true } diff --git a/tests/tests/rpc.rs b/tests/tests/rpc.rs index c0416784573..18efca1d964 100644 --- a/tests/tests/rpc.rs +++ b/tests/tests/rpc.rs @@ -1,12 +1,11 @@ #![allow(non_snake_case)] -use aws_sdk_s3::{ - Client, - config::{ - Credentials, - Region, - }, +use aws_config::{ + BehaviorVersion, + default_provider::credentials::DefaultCredentialsChain, }; +use aws_sdk_s3::Client; +use flate2::read::GzDecoder; use fuel_core::{ database::Database, service::{ @@ -16,20 +15,18 @@ use fuel_core::{ }; use fuel_core_block_aggregator_api::{ blocks::importer_and_db_source::serializer_adapter::proto_to_fuel_conversions::fuel_block_from_protobuf, - db::{ - remote_cache::block_height_to_key, - storage_or_remote_db::get_env_vars, - }, + db::remote_cache::block_height_to_key, + integration::StorageMethod, protobuf_types::{ Block as ProtoBlock, BlockHeightRequest as ProtoBlockHeightRequest, BlockRangeRequest as ProtoBlockRangeRequest, NewBlockSubscriptionRequest as ProtoNewBlockSubscriptionRequest, - RemoteBlockRangeResponse as ProtoRemoteBlockRangeResponse, - block::VersionedBlock as ProtoVersionedBlock, + RemoteBlockResponse as ProtoRemoteBlockResponse, + RemoteS3Bucket, block_aggregator_client::BlockAggregatorClient as ProtoBlockAggregatorClient, block_response::Payload as ProtoPayload, - header::VersionedHeader as ProtoVersionedHeader, + remote_block_response::Location, }, }; use fuel_core_client::client::FuelClient; @@ -39,10 +36,26 @@ use fuel_core_types::{ }; use futures::StreamExt; use prost::bytes::Bytes; -use std::borrow::Cow; +use std::io::Read; use test_helpers::client_ext::ClientExt; use tokio::time::sleep; +macro_rules! require_env_var_or_skip { + ($($var:literal),+) => { + $(if std::env::var($var).is_err() { + eprintln!("Skipping test: missing {}", $var); + return; + })+ + }; +} + +pub fn get_env_vars() -> Option<(String, String, String)> { + let aws_id = std::env::var("AWS_ACCESS_KEY_ID").ok()?; + let aws_secret = std::env::var("AWS_SECRET_ACCESS_KEY").ok()?; + let aws_region = std::env::var("AWS_REGION").ok()?; + Some((aws_id, aws_secret, aws_region)) +} + #[tokio::test(flavor = "multi_thread")] async fn get_block_range__can_get_serialized_block_from_rpc__literal() { if env_vars_are_set() { @@ -66,16 +79,9 @@ async fn get_block_range__can_get_serialized_block_from_rpc__literal() { .await .expect("could not connect to server"); - let expected_block = graphql_client - .full_block_by_height(1) - .await - .unwrap() - .unwrap(); - let expected_header = expected_block.header; - // when let request = ProtoBlockRangeRequest { start: 1, end: 1 }; - let actual_block = if let Some(ProtoPayload::Literal(block)) = rpc_client + let proto_block = if let Some(ProtoPayload::Literal(block)) = rpc_client .get_block_range(request) .await .unwrap() @@ -90,26 +96,46 @@ async fn get_block_range__can_get_serialized_block_from_rpc__literal() { } else { panic!("expected literal block payload"); }; - let ProtoVersionedBlock::V1(v1_block) = actual_block.versioned_block.unwrap(); - let actual_height = match v1_block.header.unwrap().versioned_header.unwrap() { - ProtoVersionedHeader::V1(v1_header) => v1_header.height, - ProtoVersionedHeader::V2(v2_header) => v2_header.height, - }; + let (actual_block, receipts) = + fuel_block_from_protobuf(proto_block, &[], Bytes32::default()).unwrap(); + let actual_height = actual_block.header().height(); // then - assert_eq!(expected_header.height.0, actual_height); + let expected_height = BlockHeight::new(1); + assert_eq!(&expected_height, actual_height); + + assert!( + matches!( + receipts[1], + Receipt::ScriptResult { + result: ScriptExecutionResult::Success, + .. + } + ), + "should have a script result receipt, received: {:?}", + receipts + ); + assert!( + matches!(receipts[0], Receipt::Return { .. }), + "should have a return receipt, received: {:?}", + receipts + ); } #[tokio::test(flavor = "multi_thread")] async fn get_block_range__can_get_serialized_block_from_rpc__remote() { - let Some((_, _, aws_region, aws_bucket, url_base, _)) = get_env_vars() else { - tracing::info!("Skipping test: AWS credentials are not set"); - return; - }; + require_env_var_or_skip!("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"); + ensure_bucket_exists().await; clean_s3_bucket().await; - let config = Config::local_node(); + let mut config = Config::local_node(); + let endpoint_url = "http://127.0.0.1:4566".to_string(); + config.rpc_config.storage_method = StorageMethod::S3 { + bucket: "test-bucket".to_string(), + endpoint_url: Some(endpoint_url), + requester_pays: false, + }; let rpc_url = config.rpc_config.addr; let srv = FuelService::from_database(Database::default(), config.clone()) @@ -154,11 +180,13 @@ async fn get_block_range__can_get_serialized_block_from_rpc__remote() { // then let key = block_height_to_key(&expected_height); - let expected = ProtoRemoteBlockRangeResponse { - region: aws_region.clone(), - bucket: aws_bucket.clone(), - key: key.clone(), - url: format!("{}/{}", url_base, key), + let expected = ProtoRemoteBlockResponse { + location: Some(Location::S3(RemoteS3Bucket { + bucket: "test-bucket".to_string(), + key, + requester_pays: false, + endpoint: Some("http://127.0.0.1:4566".to_string()), + })), }; assert_eq!(expected, remote_info); clean_s3_bucket().await; @@ -166,11 +194,17 @@ async fn get_block_range__can_get_serialized_block_from_rpc__remote() { #[tokio::test(flavor = "multi_thread")] async fn get_block_height__can_get_value_from_rpc() { + let mut config = Config::local_node(); if get_env_vars().is_some() { ensure_bucket_exists().await; clean_s3_bucket().await; + let endpoint_url = "http://127.0.0.1:4566".to_string(); + config.rpc_config.storage_method = StorageMethod::S3 { + bucket: "test-bucket".to_string(), + endpoint_url: Some(endpoint_url), + requester_pays: false, + }; } - let config = Config::local_node(); let rpc_url = config.rpc_config.addr; // given @@ -193,7 +227,7 @@ async fn get_block_height__can_get_value_from_rpc() { let request = ProtoBlockHeightRequest {}; let expected_height = Some(1); let actual_height = rpc_client - .get_block_height(request) + .get_synced_block_height(request) .await .unwrap() .into_inner() @@ -210,11 +244,18 @@ async fn get_block_height__can_get_value_from_rpc() { #[tokio::test(flavor = "multi_thread")] async fn new_block_subscription__can_get_expect_block() { + let mut config = Config::local_node(); if get_env_vars().is_some() { ensure_bucket_exists().await; clean_s3_bucket().await; + let endpoint_url = "http://127.0.0.1:4566".to_string(); + config.rpc_config.storage_method = StorageMethod::S3 { + bucket: "test-bucket".to_string(), + endpoint_url: Some(endpoint_url), + requester_pays: false, + }; } - let config = Config::local_node(); + let rpc_url = config.rpc_config.addr; let srv = FuelService::from_database(Database::default(), config.clone()) @@ -242,73 +283,67 @@ async fn new_block_subscription__can_get_expect_block() { let next = tokio::time::timeout(std::time::Duration::from_secs(1), stream.next()) .await .unwrap(); - let actual_block = + let proto_block = if let Some(ProtoPayload::Literal(block)) = next.unwrap().unwrap().payload { block } else { panic!("expected literal block payload"); }; - let ProtoVersionedBlock::V1(v1_block) = actual_block.versioned_block.unwrap(); - let actual_height = match v1_block.header.unwrap().versioned_header.unwrap() { - ProtoVersionedHeader::V1(v1_header) => v1_header.height, - ProtoVersionedHeader::V2(v2_header) => v2_header.height, - }; + let (actual_block, receipts) = + fuel_block_from_protobuf(proto_block, &[], Bytes32::default()).unwrap(); + let actual_height = actual_block.header().height(); // then - let expected_height = 1; - assert_eq!(expected_height, actual_height); + let expected_height = BlockHeight::new(1); + assert_eq!(&expected_height, actual_height); + + assert!( + matches!( + receipts[1], + Receipt::ScriptResult { + result: ScriptExecutionResult::Success, + .. + } + ), + "should have a script result receipt, received: {:?}", + receipts + ); + assert!( + matches!(receipts[0], Receipt::Return { .. }), + "should have a return receipt, received: {:?}", + receipts + ); + if get_env_vars().is_some() { clean_s3_bucket().await; } } -macro_rules! require_env_var_or_skip { - ($($var:literal),+) => { - $(if std::env::var($var).is_err() { - eprintln!("Skipping test: missing {}", $var); - return; - })+ - }; -} - fn env_vars_are_set() -> bool { std::env::var("AWS_ACCESS_KEY_ID").is_ok() && std::env::var("AWS_SECRET_ACCESS_KEY").is_ok() - && std::env::var("AWS_REGION").is_ok() - && std::env::var("AWS_BUCKET").is_ok() - && std::env::var("AWS_ENDPOINT_URL").is_ok() - && std::env::var("BUCKET_URL_BASE").is_ok() } -fn aws_client() -> Client { - let (aws_access_key_id, aws_secret_access_key, aws_region, _, _, aws_endpoint_url) = - get_env_vars().unwrap(); - - let mut builder = aws_sdk_s3::config::Builder::new(); - if let Some(aws_endpoint_url) = aws_endpoint_url { - builder.set_endpoint_url(Some(aws_endpoint_url.clone())); - } - - let config = builder - .force_path_style(true) - .region(Region::new(Cow::Owned(aws_region.clone()))) - .credentials_provider(Credentials::new( - aws_access_key_id, - aws_secret_access_key, - None, - None, - "block-aggregator", - )) - .behavior_version_latest() - .build(); - aws_sdk_s3::Client::from_conf(config) +async fn aws_client() -> Client { + let credentials = DefaultCredentialsChain::builder().build().await; + let _aws_region = + std::env::var("AWS_REGION").expect("AWS_REGION env var must be set"); + let sdk_config = aws_config::defaults(BehaviorVersion::latest()) + .credentials_provider(credentials) + .endpoint_url("http://127.0.0.1:4566") + .load() + .await; + let builder = aws_sdk_s3::config::Builder::from(&sdk_config); + let config = builder.force_path_style(true).build(); + Client::from_conf(config) } async fn get_block_from_s3_bucket() -> Bytes { - let client = aws_client(); - let bucket = std::env::var("AWS_BUCKET").unwrap(); + let client = aws_client().await; + let bucket = "test-bucket".to_string(); let key = block_height_to_key(&BlockHeight::new(1)); + tracing::info!("getting block from bucket: {} with key {}", bucket, key); let req = client.get_object().bucket(&bucket).key(&key); let obj = req.send().await.unwrap(); let message = format!( @@ -319,20 +354,20 @@ async fn get_block_from_s3_bucket() -> Bytes { } async fn ensure_bucket_exists() { - let client = aws_client(); - let bucket = std::env::var("AWS_BUCKET").unwrap(); - let req = client.create_bucket().bucket(&bucket); + let client = aws_client().await; + let bucket = "test-bucket"; + let req = client.create_bucket().bucket(bucket); let expect_message = format!("should be able to create bucket: {}", bucket); let _ = req.send().await.expect(&expect_message); } async fn clean_s3_bucket() { - let client = aws_client(); - let bucket = std::env::var("AWS_BUCKET").unwrap(); - let req = client.list_objects().bucket(&bucket); + let client = aws_client().await; + let bucket = "test-bucket"; + let req = client.list_objects().bucket(bucket); let objs = req.send().await.unwrap(); for obj in objs.contents.unwrap_or_default() { - let req = client.delete_object().bucket(&bucket).key(obj.key.unwrap()); + let req = client.delete_object().bucket(bucket).key(obj.key.unwrap()); let _ = req.send().await.unwrap(); } } @@ -342,19 +377,19 @@ async fn get_block_range__can_get_from_remote_s3_bucket() { let _ = tracing_subscriber::fmt() .with_max_level(tracing::Level::INFO) .try_init(); - require_env_var_or_skip!( - "AWS_ACCESS_KEY_ID", - "AWS_SECRET_ACCESS_KEY", - "AWS_REGION", - "AWS_BUCKET", - "AWS_ENDPOINT_URL", - "BUCKET_URL_BASE" - ); + + require_env_var_or_skip!("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION"); ensure_bucket_exists().await; clean_s3_bucket().await; // given - let config = Config::local_node(); + let mut config = Config::local_node(); + let endpoint_url = "http://127.0.0.1:4566".to_string(); + config.rpc_config.storage_method = StorageMethod::S3 { + bucket: "test-bucket".to_string(), + endpoint_url: Some(endpoint_url), + requester_pays: false, + }; let srv = FuelService::from_database(Database::default(), config.clone()) .await .unwrap(); @@ -367,10 +402,28 @@ async fn get_block_range__can_get_from_remote_s3_bucket() { sleep(std::time::Duration::from_secs(1)).await; // then - let data = get_block_from_s3_bucket().await; + let zipped_data = get_block_from_s3_bucket().await; + let data = unzip_bytes(&zipped_data); // can deserialize let actual_proto: ProtoBlock = prost::Message::decode(data.as_ref()).unwrap(); - let _ = fuel_block_from_protobuf(actual_proto, &[], Bytes32::default()).unwrap(); + let (_, receipts) = + fuel_block_from_protobuf(actual_proto, &[], Bytes32::default()).unwrap(); + assert!( + matches!( + receipts[1], + Receipt::ScriptResult { + result: ScriptExecutionResult::Success, + .. + } + ), + "should have a script result receipt, received: {:?}", + receipts + ); + assert!( + matches!(receipts[0], Receipt::Return { .. }), + "should have a return receipt, received: {:?}", + receipts + ); // cleanup clean_s3_bucket().await; @@ -379,3 +432,10 @@ async fn get_block_range__can_get_from_remote_s3_bucket() { "Successfully ran test: get_block_range__can_get_from_remote_s3_bucket" ); } + +fn unzip_bytes(bytes: &[u8]) -> Vec { + let mut decoder = GzDecoder::new(bytes); + let mut output = Vec::new(); + decoder.read_to_end(&mut output).unwrap(); + output +} diff --git a/tests/tests/trigger_integration/interval.rs b/tests/tests/trigger_integration/interval.rs index ad238fa8403..73a2fa94d78 100644 --- a/tests/tests/trigger_integration/interval.rs +++ b/tests/tests/trigger_integration/interval.rs @@ -160,7 +160,7 @@ async fn poa_interval_produces_nonempty_blocks_at_correct_rate() { let count_now = resp.results.len(); if count_now > count_start + rounds { - break + break; } }