diff --git a/src/chain/bitcoind.rs b/src/chain/bitcoind.rs index a120f8253..e3c38c598 100644 --- a/src/chain/bitcoind.rs +++ b/src/chain/bitcoind.rs @@ -370,8 +370,11 @@ impl BitcoindChainSource { let cur_height = channel_manager.current_best_block().height; let now = SystemTime::now(); - let unconfirmed_txids = self.onchain_wallet.get_unconfirmed_txids(); - match self.api_client.get_updated_mempool_transactions(cur_height, unconfirmed_txids).await + let bdk_unconfirmed_txids = self.onchain_wallet.get_unconfirmed_txids(); + match self + .api_client + .get_updated_mempool_transactions(cur_height, bdk_unconfirmed_txids) + .await { Ok((unconfirmed_txs, evicted_txids)) => { log_trace!( @@ -754,7 +757,7 @@ impl BitcoindClient { async fn get_raw_transaction_rpc( rpc_client: Arc, txid: &Txid, ) -> std::io::Result> { - let txid_hex = bitcoin::consensus::encode::serialize_hex(txid); + let txid_hex = txid.to_string(); let txid_json = serde_json::json!(txid_hex); match rpc_client .call_method::("getrawtransaction", &[txid_json]) @@ -792,7 +795,7 @@ impl BitcoindClient { async fn get_raw_transaction_rest( rest_client: Arc, txid: &Txid, ) -> std::io::Result> { - let txid_hex = bitcoin::consensus::encode::serialize_hex(txid); + let txid_hex = txid.to_string(); let tx_path = format!("tx/{}.json", txid_hex); match rest_client .request_resource::(&tx_path) @@ -889,7 +892,7 @@ impl BitcoindClient { async fn get_mempool_entry_inner( client: Arc, txid: Txid, ) -> std::io::Result> { - let txid_hex = bitcoin::consensus::encode::serialize_hex(&txid); + let txid_hex = txid.to_string(); let txid_json = serde_json::json!(txid_hex); match client.call_method::("getmempoolentry", &[txid_json]).await { @@ -964,11 +967,12 @@ impl BitcoindClient { /// - mempool transactions, alongside their first-seen unix timestamps. /// - transactions that have been evicted from the mempool, alongside the last time they were seen absent. pub(crate) async fn get_updated_mempool_transactions( - &self, best_processed_height: u32, unconfirmed_txids: Vec, + &self, best_processed_height: u32, bdk_unconfirmed_txids: Vec, ) -> std::io::Result<(Vec<(Transaction, u64)>, Vec<(Txid, u64)>)> { let mempool_txs = self.get_mempool_transactions_and_timestamp_at_height(best_processed_height).await?; - let evicted_txids = self.get_evicted_mempool_txids_and_timestamp(unconfirmed_txids).await?; + let evicted_txids = + self.get_evicted_mempool_txids_and_timestamp(bdk_unconfirmed_txids).await?; Ok((mempool_txs, evicted_txids)) } @@ -1078,14 +1082,14 @@ impl BitcoindClient { // To this end, we first update our local mempool_entries_cache and then return all unconfirmed // wallet `Txid`s that don't appear in the mempool still. async fn get_evicted_mempool_txids_and_timestamp( - &self, unconfirmed_txids: Vec, + &self, bdk_unconfirmed_txids: Vec, ) -> std::io::Result> { match self { BitcoindClient::Rpc { latest_mempool_timestamp, mempool_entries_cache, .. } => { Self::get_evicted_mempool_txids_and_timestamp_inner( latest_mempool_timestamp, mempool_entries_cache, - unconfirmed_txids, + bdk_unconfirmed_txids, ) .await }, @@ -1093,7 +1097,7 @@ impl BitcoindClient { Self::get_evicted_mempool_txids_and_timestamp_inner( latest_mempool_timestamp, mempool_entries_cache, - unconfirmed_txids, + bdk_unconfirmed_txids, ) .await }, @@ -1103,13 +1107,13 @@ impl BitcoindClient { async fn get_evicted_mempool_txids_and_timestamp_inner( latest_mempool_timestamp: &AtomicU64, mempool_entries_cache: &tokio::sync::Mutex>, - unconfirmed_txids: Vec, + bdk_unconfirmed_txids: Vec, ) -> std::io::Result> { let latest_mempool_timestamp = latest_mempool_timestamp.load(Ordering::Relaxed); let mempool_entries_cache = mempool_entries_cache.lock().await; - let evicted_txids = unconfirmed_txids + let evicted_txids = bdk_unconfirmed_txids .into_iter() - .filter(|txid| mempool_entries_cache.contains_key(txid)) + .filter(|txid| !mempool_entries_cache.contains_key(txid)) .map(|txid| (txid, latest_mempool_timestamp)) .collect(); Ok(evicted_txids) @@ -1236,7 +1240,7 @@ impl TryInto for JsonResponse { for hex in res { let txid = if let Some(hex_str) = hex.as_str() { - match bitcoin::consensus::encode::deserialize_hex(hex_str) { + match hex_str.parse::() { Ok(txid) => txid, Err(_) => { return Err(std::io::Error::new( @@ -1407,3 +1411,164 @@ impl std::fmt::Display for HttpError { write!(f, "status_code: {}, contents: {}", self.status_code, contents) } } + +#[cfg(test)] +mod tests { + use bitcoin::hashes::Hash; + use bitcoin::{FeeRate, OutPoint, ScriptBuf, Transaction, TxIn, TxOut, Txid, Witness}; + use lightning_block_sync::http::JsonResponse; + use proptest::{arbitrary::any, collection::vec, prop_assert_eq, prop_compose, proptest}; + use serde_json::json; + + use crate::chain::bitcoind::{ + FeeResponse, GetMempoolEntryResponse, GetRawMempoolResponse, GetRawTransactionResponse, + MempoolMinFeeResponse, + }; + + prop_compose! { + fn arbitrary_witness()( + witness_elements in vec(vec(any::(), 0..100), 0..20) + ) -> Witness { + let mut witness = Witness::new(); + for element in witness_elements { + witness.push(element); + } + witness + } + } + + prop_compose! { + fn arbitrary_txin()( + outpoint_hash in any::<[u8; 32]>(), + outpoint_vout in any::(), + script_bytes in vec(any::(), 0..100), + witness in arbitrary_witness(), + sequence in any::() + ) -> TxIn { + TxIn { + previous_output: OutPoint { + txid: Txid::from_byte_array(outpoint_hash), + vout: outpoint_vout, + }, + script_sig: ScriptBuf::from_bytes(script_bytes), + sequence: bitcoin::Sequence::from_consensus(sequence), + witness, + } + } + } + + prop_compose! { + fn arbitrary_txout()( + value in 0u64..21_000_000_00_000_000u64, + script_bytes in vec(any::(), 0..100) + ) -> TxOut { + TxOut { + value: bitcoin::Amount::from_sat(value), + script_pubkey: ScriptBuf::from_bytes(script_bytes), + } + } + } + + prop_compose! { + fn arbitrary_transaction()( + version in any::(), + inputs in vec(arbitrary_txin(), 1..20), + outputs in vec(arbitrary_txout(), 1..20), + lock_time in any::() + ) -> Transaction { + Transaction { + version: bitcoin::transaction::Version(version), + input: inputs, + output: outputs, + lock_time: bitcoin::absolute::LockTime::from_consensus(lock_time), + } + } + } + + proptest! { + #![proptest_config(proptest::test_runner::Config::with_cases(250))] + + #[test] + fn prop_get_raw_mempool_response_roundtrip(txids in vec(any::<[u8;32]>(), 0..10)) { + let txid_vec: Vec = txids.into_iter().map(Txid::from_byte_array).collect(); + let original = GetRawMempoolResponse(txid_vec.clone()); + + let json_vec: Vec = txid_vec.iter().map(|t| t.to_string()).collect(); + let json_val = serde_json::Value::Array(json_vec.iter().map(|s| json!(s)).collect()); + + let resp = JsonResponse(json_val); + let decoded: GetRawMempoolResponse = resp.try_into().unwrap(); + + prop_assert_eq!(original.0.len(), decoded.0.len()); + + prop_assert_eq!(original.0, decoded.0); + } + + #[test] + fn prop_get_mempool_entry_response_roundtrip( + time in any::(), + height in any::() + ) { + let json_val = json!({ + "time": time, + "height": height + }); + + let resp = JsonResponse(json_val); + let decoded: GetMempoolEntryResponse = resp.try_into().unwrap(); + + prop_assert_eq!(decoded.time, time); + prop_assert_eq!(decoded.height, height); + } + + #[test] + fn prop_get_raw_transaction_response_roundtrip(tx in arbitrary_transaction()) { + let hex = bitcoin::consensus::encode::serialize_hex(&tx); + let json_val = serde_json::Value::String(hex.clone()); + + let resp = JsonResponse(json_val); + let decoded: GetRawTransactionResponse = resp.try_into().unwrap(); + + prop_assert_eq!(decoded.0.compute_txid(), tx.compute_txid()); + prop_assert_eq!(decoded.0.compute_wtxid(), tx.compute_wtxid()); + + prop_assert_eq!(decoded.0, tx); + } + + #[test] + fn prop_fee_response_roundtrip(fee_rate in any::()) { + let fee_rate = fee_rate.abs(); + let json_val = json!({ + "feerate": fee_rate, + "errors": serde_json::Value::Null + }); + + let resp = JsonResponse(json_val); + let decoded: FeeResponse = resp.try_into().unwrap(); + + let expected = { + let fee_rate_sat_per_kwu = (fee_rate * 25_000_000.0).round() as u64; + FeeRate::from_sat_per_kwu(fee_rate_sat_per_kwu) + }; + prop_assert_eq!(decoded.0, expected); + } + + #[test] + fn prop_mempool_min_fee_response_roundtrip(fee_rate in any::()) { + let fee_rate = fee_rate.abs(); + let json_val = json!({ + "mempoolminfee": fee_rate + }); + + let resp = JsonResponse(json_val); + let decoded: MempoolMinFeeResponse = resp.try_into().unwrap(); + + let expected = { + let fee_rate_sat_per_kwu = (fee_rate * 25_000_000.0).round() as u64; + FeeRate::from_sat_per_kwu(fee_rate_sat_per_kwu) + }; + prop_assert_eq!(decoded.0, expected); + } + + } +} diff --git a/tests/common/mod.rs b/tests/common/mod.rs index ab66f0fdd..8f7bced83 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -31,7 +31,7 @@ use lightning_persister::fs_store::FilesystemStore; use bitcoin::hashes::sha256::Hash as Sha256; use bitcoin::hashes::Hash; -use bitcoin::{Address, Amount, Network, OutPoint, Txid}; +use bitcoin::{Address, Amount, Network, OutPoint, Transaction, Txid}; use electrsd::corepc_node::Client as BitcoindClient; use electrsd::corepc_node::Node as BitcoinD; @@ -487,12 +487,33 @@ where pub(crate) fn premine_and_distribute_funds( bitcoind: &BitcoindClient, electrs: &E, addrs: Vec
, amount: Amount, ) { + premine_blocks(bitcoind, electrs); + + distribute_funds(bitcoind, electrs, addrs, amount); +} + +pub(crate) fn premine_blocks(bitcoind: &BitcoindClient, electrs: &E) { let _ = bitcoind.create_wallet("ldk_node_test"); let _ = bitcoind.load_wallet("ldk_node_test"); generate_blocks_and_wait(bitcoind, electrs, 101); +} - let amounts: HashMap = - addrs.iter().map(|addr| (addr.to_string(), amount.to_btc())).collect(); +pub(crate) fn distribute_funds( + bitcoind: &BitcoindClient, electrs: &E, addrs: Vec
, amount: Amount, +) -> Txid { + let address_txid_map = distribute_funds_unconfirmed(bitcoind, electrs, addrs, amount); + generate_blocks_and_wait(bitcoind, electrs, 1); + + address_txid_map +} + +pub(crate) fn distribute_funds_unconfirmed( + bitcoind: &BitcoindClient, electrs: &E, addrs: Vec
, amount: Amount, +) -> Txid { + let mut amounts = HashMap::::new(); + for addr in &addrs { + amounts.insert(addr.to_string(), amount.to_btc()); + } let empty_account = json!(""); let amounts_json = json!(amounts); @@ -505,7 +526,12 @@ pub(crate) fn premine_and_distribute_funds( .unwrap(); wait_for_tx(electrs, txid); - generate_blocks_and_wait(bitcoind, electrs, 1); + + txid +} + +pub(crate) fn get_transaction(electrs: &E, txid: Txid) -> Transaction { + electrs.transaction_get(&txid).unwrap() } pub fn open_channel( diff --git a/tests/integration_tests_rust.rs b/tests/integration_tests_rust.rs index ad3867429..bc7f5271a 100644 --- a/tests/integration_tests_rust.rs +++ b/tests/integration_tests_rust.rs @@ -10,6 +10,7 @@ mod common; use common::{ do_channel_full_cycle, expect_channel_pending_event, expect_channel_ready_event, expect_event, expect_payment_received_event, expect_payment_successful_event, generate_blocks_and_wait, + get_transaction, logging::{init_log_logger, validate_log_entry, TestLogWriter}, open_channel, premine_and_distribute_funds, random_config, random_listening_addresses, setup_bitcoind_and_electrsd, setup_builder, setup_node, setup_two_nodes, wait_for_tx, @@ -32,15 +33,18 @@ use lightning_invoice::{Bolt11InvoiceDescription, Description}; use lightning_types::payment::PaymentPreimage; use bitcoin::address::NetworkUnchecked; +use bitcoin::hashes::hex::FromHex; use bitcoin::hashes::sha256::Hash as Sha256Hash; use bitcoin::hashes::Hash; -use bitcoin::Address; -use bitcoin::Amount; +use bitcoin::{Address, Amount, ScriptBuf, Sequence, Transaction, Witness}; use log::LevelFilter; +use std::collections::HashSet; use std::str::FromStr; use std::sync::Arc; +use crate::common::{distribute_funds_unconfirmed, premine_blocks}; + #[test] fn channel_full_cycle() { let (bitcoind, electrsd) = setup_bitcoind_and_electrsd(); @@ -669,6 +673,219 @@ fn onchain_wallet_recovery() { ); } +#[test] +fn test_rbf_via_mempool() { + run_rbf_test(false); +} + +#[test] +fn test_rbf_via_direct_block_insertion() { + run_rbf_test(true); +} + +// `is_insert_block`: +// - `true`: transaction is mined immediately (no mempool), testing confirmed-Tx handling. +// - `false`: transaction stays in mempool until confirmation, testing unconfirmed-Tx handling. +fn run_rbf_test(is_insert_block: bool) { + let (bitcoind, electrsd) = setup_bitcoind_and_electrsd(); + let chain_source_bitcoind = TestChainSource::BitcoindRpcSync(&bitcoind); + let chain_source_electrsd = TestChainSource::Electrum(&electrsd); + let chain_source_esplora = TestChainSource::Esplora(&electrsd); + + macro_rules! config_node { + ($chain_source: expr, $anchor_channels: expr) => {{ + let config_a = random_config($anchor_channels); + let node = setup_node(&$chain_source, config_a, None); + node + }}; + } + let anchor_channels = false; + let nodes = vec![ + config_node!(chain_source_electrsd, anchor_channels), + config_node!(chain_source_bitcoind, anchor_channels), + config_node!(chain_source_esplora, anchor_channels), + ]; + + let (bitcoind, electrs) = (&bitcoind.client, &electrsd.client); + premine_blocks(bitcoind, electrs); + + // Helpers declaration before starting the test + let all_addrs = + nodes.iter().map(|node| node.onchain_payment().new_address().unwrap()).collect::>(); + let amount_sat = 2_100_000; + let mut txid; + macro_rules! distribute_funds_all_nodes { + () => { + txid = distribute_funds_unconfirmed( + bitcoind, + electrs, + all_addrs.clone(), + Amount::from_sat(amount_sat), + ); + }; + } + + let mut tx; + let scripts_buf: HashSet = + all_addrs.iter().map(|addr| addr.script_pubkey()).collect(); + let mut fee_output_index; + macro_rules! prepare_rbf { + () => { + tx = get_transaction(electrs, txid); + + let mut option_fee_output_index = None; + for (index, output) in tx.output.iter().enumerate() { + if !scripts_buf.contains(&output.script_pubkey) { + option_fee_output_index = Some(index); + break; + } + } + fee_output_index = option_fee_output_index.expect( + "No output available for fee pumping. Need at least one output not being modified.", + ); + }; + } + + let mut bump_fee_amount_sat; + macro_rules! bump_fee_rbf_and_public_transaction { + () => { + bump_fee_amount_sat = tx.vsize() as u64; + let attempts = 5; + for _attempt in 0..attempts { + bump_fee!(); + println!("Bumping fee to {} sats", bump_fee_amount_sat); + println!("Transaction ID: {}", tx.compute_txid()); + match bitcoind.send_raw_transaction(&tx) { + Ok(res) => { + // Mine a block immediately so the transaction is confirmed + // before any node identifies it as a transaction that was in the mempool. + if is_insert_block { + generate_blocks_and_wait(bitcoind, electrs, 1); + } + let new_txid = res.0.parse().unwrap(); + wait_for_tx(electrs, new_txid); + break; + }, + Err(_) => { + if _attempt == attempts - 1 { + panic!("Failed to pump fee after {} attempts", attempts); + } + + bump_fee_amount_sat += bump_fee_amount_sat * 5; + if tx.output[fee_output_index].value.to_sat() < bump_fee_amount_sat { + panic!("Insufficient funds to increase fee"); + } + }, + } + } + }; + } + + macro_rules! bump_fee { + () => { + let fee_output = &mut tx.output[fee_output_index]; + let new_fee_value = fee_output.value.to_sat().saturating_sub(bump_fee_amount_sat); + fee_output.value = Amount::from_sat(new_fee_value); + println!("New fee value: {} sats", new_fee_value); + + // dust limit + if new_fee_value < 546 { + panic!("Warning: Fee output approaching dust limit ({} sats)", new_fee_value); + } + + for input in &mut tx.input { + input.sequence = Sequence::ENABLE_RBF_NO_LOCKTIME; + input.script_sig = ScriptBuf::new(); + input.witness = Witness::new(); + } + + let signed_result = bitcoind.sign_raw_transaction_with_wallet(&tx).unwrap(); + assert!(signed_result.complete, "Failed to sign RBF transaction"); + + let tx_bytes = Vec::::from_hex(&signed_result.hex).unwrap(); + tx = bitcoin::consensus::encode::deserialize::(&tx_bytes).unwrap(); + }; + } + + macro_rules! validate_balances { + ($expected_balance_sat: expr, $is_spendable: expr) => { + let spend_balance = if $is_spendable { $expected_balance_sat } else { 0 }; + for node in &nodes { + node.sync_wallets().unwrap(); + let balances = node.list_balances(); + assert_eq!(balances.spendable_onchain_balance_sats, spend_balance); + assert_eq!(balances.total_onchain_balance_sats, $expected_balance_sat); + } + }; + } + + // Modify the output to the nodes + distribute_funds_all_nodes!(); + validate_balances!(amount_sat, false); + prepare_rbf!(); + tx.output.iter_mut().for_each(|output| { + if scripts_buf.contains(&output.script_pubkey) { + let new_addr = bitcoind.new_address().unwrap(); + output.script_pubkey = new_addr.script_pubkey(); + } + }); + bump_fee_rbf_and_public_transaction!(); + validate_balances!(0, is_insert_block); + + // Not modifying the output scripts, but still bumping the fee. + distribute_funds_all_nodes!(); + validate_balances!(amount_sat, false); + prepare_rbf!(); + bump_fee_rbf_and_public_transaction!(); + validate_balances!(amount_sat, is_insert_block); + + let mut final_amount_sat = amount_sat * 2; + let value_sat = 21_000; + + // Increase the value of the nodes' outputs + distribute_funds_all_nodes!(); + prepare_rbf!(); + tx.output.iter_mut().for_each(|output| { + if scripts_buf.contains(&output.script_pubkey) { + output.value = Amount::from_sat(output.value.to_sat() + value_sat); + } + }); + bump_fee_rbf_and_public_transaction!(); + final_amount_sat += value_sat; + validate_balances!(final_amount_sat, is_insert_block); + + // Decreases the value of the nodes' outputs + distribute_funds_all_nodes!(); + final_amount_sat += amount_sat; + prepare_rbf!(); + tx.output.iter_mut().for_each(|output| { + if scripts_buf.contains(&output.script_pubkey) { + output.value = Amount::from_sat(output.value.to_sat() - value_sat); + } + }); + bump_fee_rbf_and_public_transaction!(); + final_amount_sat -= value_sat; + validate_balances!(final_amount_sat, is_insert_block); + + if !is_insert_block { + generate_blocks_and_wait(bitcoind, electrs, 1); + validate_balances!(final_amount_sat, true); + } + + // Check if it is possible to send all funds from the node + let mut txids = Vec::new(); + let addr = bitcoind.new_address().unwrap(); + nodes.iter().for_each(|node| { + let txid = node.onchain_payment().send_all_to_address(&addr, true, None).unwrap(); + txids.push(txid); + }); + txids.iter().for_each(|txid| { + wait_for_tx(electrs, *txid); + }); + generate_blocks_and_wait(bitcoind, electrs, 6); + validate_balances!(0, true); +} + #[test] fn sign_verify_msg() { let (_bitcoind, electrsd) = setup_bitcoind_and_electrsd();