Add manual block generation and direct transaction insertion for RBF testing

- Introduce `generate_block_and_insert_transactions` to manually mine a block with arbitrary transactions.
- Update `bump_fee_and_broadcast` to optionally insert transactions directly into a block (bypassing the mempool) when `is_insert_block` is true.
- Add integration test to insert the original (pre-RBF) transaction into a block instead of the RBF, to cover scenarios where the original transaction is confirmed and the RBF is not.

Author: moisesPompilio

tests/common/mod.rs

@@ -16,15 +16,22 @@ use std::env;
 use std::future::Future;
 use std::path::PathBuf;
 use std::pin::Pin;
+use std::str::FromStr;
 use std::sync::{Arc, RwLock};
 use std::time::Duration;
 
-use bitcoin::hashes::hex::FromHex;
-use bitcoin::hashes::sha256::Hash as Sha256;
-use bitcoin::hashes::Hash;
+use bitcoin::absolute::LockTime;
+use bitcoin::block::{Header, Version as BlockVersion};
+use bitcoin::hashes::{hex::FromHex, sha256::Hash as Sha256, Hash};
+use bitcoin::merkle_tree::calculate_root;
+use bitcoin::opcodes::all::OP_RETURN;
+use bitcoin::script::Builder as BuilderScriptBitcoin;
+use bitcoin::transaction::Version;
 use bitcoin::{
-	Address, Amount, Network, OutPoint, ScriptBuf, Sequence, Transaction, Txid, Witness,
+	Address, Amount, Block, BlockHash, CompactTarget, Network, OutPoint, ScriptBuf, Sequence,
+	Transaction, TxMerkleNode, Txid, Witness, Wtxid,
 };
+use electrsd::corepc_node::TemplateRules;
 use electrsd::corepc_node::{Client as BitcoindClient, Node as BitcoinD};
 use electrsd::{corepc_node, ElectrsD};
 use electrum_client::ElectrumApi;
@@ -394,6 +401,133 @@ pub(crate) fn setup_node_for_async_payments(
 	node
 }
 
+pub(crate) fn generate_block_and_insert_transactions<E: ElectrumApi>(
+	bitcoind: &BitcoindClient, electrs: &E, txs: &[Transaction],
+) {
+	let _ = bitcoind.create_wallet("ldk_node_test");
+	let _ = bitcoind.load_wallet("ldk_node_test");
+	let blockchain_info = bitcoind.get_blockchain_info().expect("failed to get blockchain info");
+	let cur_height = blockchain_info.blocks;
+	let address = bitcoind.new_address().expect("failed to get new address");
+
+	let request_block_template =
+		corepc_node::TemplateRequest { rules: vec![TemplateRules::Segwit] };
+	let bt =
+		bitcoind.get_block_template(&request_block_template).expect("failed to get block template");
+
+	// === BIP 141: Witness Commitment Calculation ===
+	let witness_root = if txs.is_empty() {
+		TxMerkleNode::all_zeros()
+	} else {
+		// BIP 141: Create wtxid list starting with all-zeros for coinbase
+		let wtxids: Vec<Wtxid> = std::iter::once(Wtxid::all_zeros())
+			.chain(txs.iter().map(|tx| tx.compute_wtxid()))
+			.collect();
+
+		calculate_root(wtxids.into_iter())
+			.map(|root| TxMerkleNode::from_byte_array(root.to_byte_array()))
+			.unwrap()
+	};
+
+	// BIP 141: Witness reserved value (32 zero bytes)
+	let witness_reserved_value = [0u8; 32];
+
+	// BIP 141: Calculate commitment hash = Double-SHA256(witness root || witness reserved value)
+	let mut commitment_preimage = Vec::new();
+	commitment_preimage.extend_from_slice(&witness_root.to_byte_array());
+	commitment_preimage.extend_from_slice(&witness_reserved_value);
+	let commitment_hash = Sha256::hash(&commitment_preimage);
+
+	// === Coinbase Transaction Construction ===
+	// BIP 141: Coinbase witness contains the witness reserved value
+	let mut coinbase_witness = Witness::new();
+	coinbase_witness.push(&witness_reserved_value);
+
+	// BIP 141: Witness commitment output script
+	let mut bip_141_data = [0u8; 36];
+	bip_141_data[0] = 0xaa;
+	bip_141_data[1] = 0x21;
+	bip_141_data[2] = 0xa9;
+	bip_141_data[3] = 0xed;
+	bip_141_data[4..].copy_from_slice(&commitment_hash.to_byte_array());
+
+	// Format: OP_RETURN + OP_PUSHBYTES_36 + 0xaa21a9ed + 32-byte commitment hash
+	let witness_commitment_script =
+		BuilderScriptBitcoin::new().push_opcode(OP_RETURN).push_slice(bip_141_data).into_script();
+
+	// BIP 34: Block height in coinbase input script
+	let block_height = bt.height;
+	let height_script = BuilderScriptBitcoin::new()
+		.push_int(block_height as i64) // BIP 34: block height as first item
+		.push_int(bdk_chain::bitcoin::secp256k1::rand::random()) // Random nonce for uniqueness
+		.into_script();
+
+	// Do not use the coinbase value from the block template.
+	// The template may include transactions not actually mined, so fees may be incorrect.
+	let coinbase_output_value = 1_250_000_000;
+
+	let coinbase_tx = Transaction {
+		version: Version::ONE,
+		lock_time: LockTime::from_height(0).unwrap(),
+		input: vec![bitcoin::TxIn {
+			previous_output: bdk_chain::bitcoin::OutPoint::default(), // Null outpoint for coinbase
+			script_sig: height_script,                                // BIP 34: height + random data
+			sequence: Sequence::default(),
+			witness: coinbase_witness, // BIP 141: witness reserved value
+		}],
+		output: vec![
+			// Coinbase reward output
+			bitcoin::TxOut {
+				value: Amount::from_sat(coinbase_output_value),
+				script_pubkey: address.script_pubkey(),
+			},
+			// BIP 141: Witness commitment output (must be last output)
+			bitcoin::TxOut { value: Amount::ZERO, script_pubkey: witness_commitment_script },
+		],
+	};
+
+	// === Block Construction ===
+	let bits_hex = bt.bits.clone();
+	let bits_vec = Vec::<u8>::from_hex(&bits_hex).expect("invalid hex for bits");
+	let bits: [u8; 4] = bits_vec.try_into().expect("bits must be 4 bytes");
+	let prev_hash_block = BlockHash::from_str(&bt.previous_block_hash).expect("invalid prev hash");
+
+	let txdata = std::iter::once(coinbase_tx).chain(txs.iter().cloned()).collect::<Vec<_>>();
+	let mut block = Block {
+		header: Header {
+			version: BlockVersion::default(),
+			prev_blockhash: prev_hash_block,
+			merkle_root: TxMerkleNode::all_zeros(), // Will be calculated below
+			time: Ord::max(
+				bt.min_time,
+				std::time::UNIX_EPOCH.elapsed().unwrap().as_secs().try_into().unwrap(),
+			) as u32,
+			bits: CompactTarget::from_consensus(u32::from_be_bytes(bits)),
+			nonce: 0,
+		},
+		txdata,
+	};
+
+	block.header.merkle_root = block.compute_merkle_root().expect("must compute");
+
+	for nonce in 0..=u32::MAX {
+		block.header.nonce = nonce;
+		if block.header.target().is_met_by(block.block_hash()) {
+			break;
+		}
+	}
+
+	match bitcoind.submit_block(&block) {
+		Ok(_) => println!("Generated block 1 manually with {} transactions", txs.len()),
+		Err(e) => panic!("Failed to submit block: {:?}", e),
+	}
+	wait_for_block(electrs, cur_height as usize + 1);
+
+	txs.iter().for_each(|tx| {
+		wait_for_tx(electrs, tx.compute_txid());
+	});
+}
+
 pub(crate) fn generate_blocks_and_wait<E: ElectrumApi>(
 	bitcoind: &BitcoindClient, electrs: &E, num: usize,
 ) {
@@ -575,11 +709,13 @@ pub(crate) fn bump_fee_and_broadcast<E: ElectrumApi>(
 		let tx_bytes = Vec::<u8>::from_hex(&signed_result.hex).unwrap();
 		tx = bitcoin::consensus::encode::deserialize::<Transaction>(&tx_bytes).unwrap();
 
+		if is_insert_block {
+			generate_block_and_insert_transactions(bitcoind, electrs, &[tx.clone()]);
+			return tx;
+		}
+
 		match bitcoind.send_raw_transaction(&tx) {
 			Ok(res) => {
-				if is_insert_block {
-					generate_blocks_and_wait(bitcoind, electrs, 1);
-				}
 				let new_txid: Txid = res.0.parse().unwrap();
 				wait_for_tx(electrs, new_txid);
 				return tx;

tests/integration_tests_rust.rs

@@ -20,10 +20,11 @@ use common::{
 	bump_fee_and_broadcast, distribute_funds_unconfirmed, do_channel_full_cycle,
 	expect_channel_pending_event, expect_channel_ready_event, expect_event,
 	expect_payment_claimable_event, expect_payment_received_event, expect_payment_successful_event,
-	generate_blocks_and_wait, open_channel, open_channel_push_amt, premine_and_distribute_funds,
-	premine_blocks, prepare_rbf, random_config, random_listening_addresses,
-	setup_bitcoind_and_electrsd, setup_builder, setup_node, setup_node_for_async_payments,
-	setup_node_from_config, setup_two_nodes, wait_for_tx, TestChainSource, TestSyncStore,
+	generate_block_and_insert_transactions, generate_blocks_and_wait, open_channel,
+	open_channel_push_amt, premine_and_distribute_funds, premine_blocks, prepare_rbf,
+	random_config, random_listening_addresses, setup_bitcoind_and_electrsd, setup_builder,
+	setup_node, setup_node_for_async_payments, setup_node_from_config, setup_two_nodes,
+	wait_for_tx, TestChainSource, TestSyncStore,
 };
 use ldk_node::config::{AsyncPaymentsRole, EsploraSyncConfig};
 use ldk_node::liquidity::LSPS2ServiceConfig;
@@ -669,19 +670,24 @@ fn onchain_wallet_recovery() {
 }
 
 #[test]
-fn test_rbf_via_mempool() {
-	run_rbf_test(false);
+fn test_rbf_only_in_mempool() {
+	run_rbf_test(false, false);
 }
 
 #[test]
-fn test_rbf_via_direct_block_insertion() {
-	run_rbf_test(true);
+fn test_rbf_direct_block_insertion_rbf_tx() {
+	run_rbf_test(true, false);
+}
+
+#[test]
+fn test_rbf_direct_block_insertion_original_tx() {
+	run_rbf_test(false, true);
 }
 
 // `is_insert_block`:
 // - `true`: transaction is mined immediately (no mempool), testing confirmed-Tx handling.
 // - `false`: transaction stays in mempool until confirmation, testing unconfirmed-Tx handling.
-fn run_rbf_test(is_insert_block: bool) {
+fn run_rbf_test(is_insert_block: bool, is_insertion_original_tx: bool) {
 	let (bitcoind, electrsd) = setup_bitcoind_and_electrsd();
 	let chain_source_bitcoind = TestChainSource::BitcoindRpcSync(&bitcoind);
 	let chain_source_electrsd = TestChainSource::Electrum(&electrsd);
@@ -724,58 +730,98 @@ fn run_rbf_test(is_insert_block: bool) {
 		};
 	}
 
+	macro_rules! validate_total_onchain_balance {
+		($expected_balance_sat: expr) => {
+			for node in &nodes {
+				node.sync_wallets().unwrap();
+				let balances = node.list_balances();
+				assert_eq!(balances.total_onchain_balance_sats, $expected_balance_sat);
+			}
+		};
+	}
+
 	let scripts_buf: HashSet<ScriptBuf> =
 		all_addrs.iter().map(|addr| addr.script_pubkey()).collect();
 	let mut tx;
 	let mut fee_output_index;
 
-	// Modify the output to the nodes
+	let mut final_amount_sat = 0;
+	let mut original_tx;
+
+	// Step 1: Bump fee and change output address
 	distribute_funds_all_nodes!();
 	validate_balances!(amount_sat, false);
 	(tx, fee_output_index) = prepare_rbf(electrs, txid, &scripts_buf);
+	original_tx = tx.clone();
 	tx.output.iter_mut().for_each(|output| {
 		if scripts_buf.contains(&output.script_pubkey) {
 			let new_addr = bitcoind.new_address().unwrap();
 			output.script_pubkey = new_addr.script_pubkey();
 		}
 	});
 	bump_fee_and_broadcast(bitcoind, electrs, tx, fee_output_index, is_insert_block);
-	validate_balances!(0, is_insert_block);
+	if is_insertion_original_tx {
+		generate_block_and_insert_transactions(bitcoind, electrs, &[original_tx.clone()]);
+	}
+	if is_insertion_original_tx {
+		final_amount_sat += amount_sat;
+	}
+	validate_balances!(final_amount_sat, is_insert_block || is_insertion_original_tx);
 
-	// Not modifying the output scripts, but still bumping the fee.
+	// Step 2: Bump fee only
 	distribute_funds_all_nodes!();
-	validate_balances!(amount_sat, false);
+	validate_total_onchain_balance!(amount_sat + final_amount_sat);
 	(tx, fee_output_index) = prepare_rbf(electrs, txid, &scripts_buf);
+	original_tx = tx.clone();
 	bump_fee_and_broadcast(bitcoind, electrs, tx, fee_output_index, is_insert_block);
-	validate_balances!(amount_sat, is_insert_block);
+	if is_insertion_original_tx {
+		generate_block_and_insert_transactions(bitcoind, electrs, &[original_tx.clone()]);
+	}
+	final_amount_sat += amount_sat;
+	validate_balances!(final_amount_sat, is_insert_block || is_insertion_original_tx);
 
-	let mut final_amount_sat = amount_sat * 2;
+	// Step 3: Increase output value
 	let value_sat = 21_000;
-
-	// Increase the value of the nodes' outputs
 	distribute_funds_all_nodes!();
+	validate_total_onchain_balance!(amount_sat + final_amount_sat);
 	(tx, fee_output_index) = prepare_rbf(electrs, txid, &scripts_buf);
+	original_tx = tx.clone();
 	tx.output.iter_mut().for_each(|output| {
 		if scripts_buf.contains(&output.script_pubkey) {
 			output.value = Amount::from_sat(output.value.to_sat() + value_sat);
 		}
 	});
+	tx.output[fee_output_index].value -= Amount::from_sat(scripts_buf.len() as u64 * value_sat);
 	bump_fee_and_broadcast(bitcoind, electrs, tx, fee_output_index, is_insert_block);
-	final_amount_sat += value_sat;
-	validate_balances!(final_amount_sat, is_insert_block);
+	if is_insertion_original_tx {
+		generate_block_and_insert_transactions(bitcoind, electrs, &[original_tx.clone()]);
+	}
+	final_amount_sat += amount_sat;
+	if !is_insertion_original_tx {
+		final_amount_sat += value_sat;
+	}
+	validate_balances!(final_amount_sat, is_insert_block || is_insertion_original_tx);
 
-	// Decreases the value of the nodes' outputs
+	// Step 4: Decrease output value
 	distribute_funds_all_nodes!();
-	final_amount_sat += amount_sat;
+	validate_total_onchain_balance!(amount_sat + final_amount_sat);
 	(tx, fee_output_index) = prepare_rbf(electrs, txid, &scripts_buf);
+	original_tx = tx.clone();
 	tx.output.iter_mut().for_each(|output| {
 		if scripts_buf.contains(&output.script_pubkey) {
 			output.value = Amount::from_sat(output.value.to_sat() - value_sat);
 		}
 	});
+	tx.output[fee_output_index].value += Amount::from_sat(scripts_buf.len() as u64 * value_sat);
 	bump_fee_and_broadcast(bitcoind, electrs, tx, fee_output_index, is_insert_block);
-	final_amount_sat -= value_sat;
-	validate_balances!(final_amount_sat, is_insert_block);
+	if is_insertion_original_tx {
+		generate_block_and_insert_transactions(bitcoind, electrs, &[original_tx.clone()]);
+	}
+	final_amount_sat += amount_sat;
+	if !is_insertion_original_tx {
+		final_amount_sat -= value_sat;
+	}
+	validate_balances!(final_amount_sat, is_insert_block || is_insertion_original_tx);
 
 	if !is_insert_block {
 		generate_blocks_and_wait(bitcoind, electrs, 1);