Skip to content

Commit 1039d3c

Browse files
committed
Merge #1957: feat(electrum): optimize merkle proof validation with batching
156cbab test(electrum): Improve benchmark (志宇) 4ea5ea6 feat(electrum): batch `transaction.get_merkle` calls via `batch_call` (Wei Chen) ec4fd97 feat(electrum): batched `Header`s and `script_get_history` (Wei Chen) f21a21d test(electrum): add `criterion` benchmark for `sync` (Wei Chen) b57768d fix(electrum): improve tx validation and gap limit scanning (keerthi) 7a18cad feat(electrum): optimize merkle proof validation with batching (Wei Chen) Pull request description: Replaces #1908, originally authored by @Keerthi421. Fixes #1891. ### Description This PR optimizes `sync`/`full_scan` performance by batching and caching key RPC calls to slash network round-trips and eliminate redundant work. Key improvements: * Gather all `blockchain.transaction.get_merkle` calls into a single `batch_call` request. * Use `batch_script_get_history` instead of many individual `script_get_history` calls. * Use `batch_block_header` to fetch all needed block headers in one call rather than repeatedly calling `block_header`. * Introduce a cache of transaction anchors to skip re-validating already confirmed transactions. #### Anchor Caching Performance Improvements Results suggest a significant speed up with a warmed up cache. Tested on local Electrum server with: ``` $ cargo bench -p bdk_electrum --bench test_sync ``` Results before this PR (https://github.com/LagginTimes/bdk/tree/1957-master-branch): ``` sync_with_electrum time: [1.3702 s 1.3732 s 1.3852 s] ``` Results after this PR: ``` sync_with_electrum time: [851.31 ms 853.26 ms 856.23 ms] ``` #### Batch Call Performance Improvements No persisted data was carried over between runs, so each test started with cold caches and measured only raw batching performance. Tested with`example_electrum` out of https://github.com/LagginTimes/bdk/tree/example_electrum_timing with the following parameters: ``` $ example_electrum init "tr([62f3f3af/86'/1'/0']tpubDD4Kse29e47rSP5paSuNPhWnGMcdEDAuiG42LEd5yaRDN2CFApWiLTAzxQSLS7MpvxrpxvRJBVcjhVPRk7gec4iWfwvLrEhns1LA4h7i3c2/0/*)#cn4sudyq" $ example_electrum scan tcp://signet-electrumx.wakiyamap.dev:50001 ``` Results before this PR: ``` FULL_SCAN TIME: 8.145874476s ``` Results after this PR (using this PR's [`bdk_electrum_client.rs`](https://github.com/bitcoindevkit/bdk/blob/70495e2010541acbb5d62f9b5692de20924ac53f/crates/electrum/src/bdk_electrum_client.rs)): ``` FULL_SCAN TIME: 2.594050112s ``` ### Changelog notice * Add transaction anchor cache to prevent redundant network calls. * Batch Merkle proof, script history, and header requests. ### Checklists #### All Submissions: * [x] I've signed all my commits * [x] I followed the [contribution guidelines](https://github.com/bitcoindevkit/bdk/blob/master/CONTRIBUTING.md) * [x] I ran `cargo fmt` and `cargo clippy` before committing #### New Features: * [x] I've added tests for the new feature * [x] I've added docs for the new feature #### Bugfixes: * [ ] This pull request breaks the existing API * [x] I've added tests to reproduce the issue which are now passing * [x] I'm linking the issue being fixed by this PR ACKs for top commit: oleonardolima: tACK 156cbab evanlinjin: ACK 156cbab Tree-SHA512: dc7dc1d7de938223cc03293d8bb8ae12c8799c7ec8ba8c7faec5cf2076c96a1b1e50b406cbcc90cbd6cbe7a311c0c11dd036691c03ed067c469a26260903993b
2 parents 63923c6 + 156cbab commit 1039d3c

File tree

3 files changed

+364
-136
lines changed

3 files changed

+364
-136
lines changed

crates/electrum/Cargo.toml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,12 @@ workspace = true
1515
[dependencies]
1616
bdk_core = { path = "../core", version = "0.6.0" }
1717
electrum-client = { version = "0.23.1", features = [ "proxy" ], default-features = false }
18+
serde_json = "1.0"
1819

1920
[dev-dependencies]
2021
bdk_testenv = { path = "../testenv" }
2122
bdk_chain = { path = "../chain" }
23+
criterion = { version = "0.2" }
2224

2325
[features]
2426
default = ["use-rustls"]
@@ -29,3 +31,7 @@ use-openssl = ["electrum-client/use-openssl"]
2931
[[test]]
3032
name = "test_electrum"
3133
required-features = ["use-rustls"]
34+
35+
[[bench]]
36+
name = "test_sync"
37+
harness = false

crates/electrum/benches/test_sync.rs

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
use bdk_chain::bitcoin::{Address, Amount, ScriptBuf};
2+
use bdk_core::{
3+
bitcoin::{
4+
consensus::WriteExt,
5+
hashes::Hash,
6+
key::{Secp256k1, UntweakedPublicKey},
7+
Network, TapNodeHash,
8+
},
9+
spk_client::SyncRequest,
10+
CheckPoint,
11+
};
12+
use bdk_electrum::BdkElectrumClient;
13+
use bdk_testenv::{anyhow, bitcoincore_rpc::RpcApi, TestEnv};
14+
use criterion::{criterion_group, criterion_main, Criterion};
15+
use electrum_client::ElectrumApi;
16+
use std::{collections::BTreeSet, time::Duration};
17+
18+
// Batch size for `sync_with_electrum`.
19+
const BATCH_SIZE: usize = 100;
20+
21+
pub fn get_test_spk(i: usize) -> ScriptBuf {
22+
const PK_BYTES: &[u8] = &[
23+
12, 244, 72, 4, 163, 4, 211, 81, 159, 82, 153, 123, 125, 74, 142, 40, 55, 237, 191, 231,
24+
31, 114, 89, 165, 83, 141, 8, 203, 93, 240, 53, 101,
25+
];
26+
let secp = Secp256k1::new();
27+
let pk = UntweakedPublicKey::from_slice(PK_BYTES).expect("Must be valid PK");
28+
let mut engine = TapNodeHash::engine();
29+
engine.emit_u64(i as u64).expect("must emit");
30+
ScriptBuf::new_p2tr(&secp, pk, Some(TapNodeHash::from_engine(engine)))
31+
}
32+
33+
fn sync_with_electrum<E: ElectrumApi>(
34+
client: &BdkElectrumClient<E>,
35+
spks: &[ScriptBuf],
36+
chain_tip: &CheckPoint,
37+
) -> anyhow::Result<()> {
38+
let update = client.sync(
39+
SyncRequest::builder()
40+
.chain_tip(chain_tip.clone())
41+
.spks(spks.iter().cloned()),
42+
BATCH_SIZE,
43+
true,
44+
)?;
45+
46+
assert!(
47+
!update.tx_update.txs.is_empty(),
48+
"expected some transactions from sync, but got none"
49+
);
50+
51+
Ok(())
52+
}
53+
54+
pub fn test_sync_performance(c: &mut Criterion) {
55+
let env = TestEnv::new().unwrap();
56+
57+
const NUM_BLOCKS: usize = 100;
58+
let mut spks = Vec::with_capacity(NUM_BLOCKS);
59+
60+
// Mine some blocks and send transactions.
61+
env.mine_blocks(101, None).unwrap();
62+
63+
// Scatter UTXOs across many blocks.
64+
for i in 0..NUM_BLOCKS {
65+
let spk = get_test_spk(i);
66+
let addr = Address::from_script(&spk, Network::Regtest).unwrap();
67+
env.send(&addr, Amount::from_sat(10_000)).unwrap();
68+
env.mine_blocks(1, None).unwrap();
69+
70+
spks.push(spk);
71+
}
72+
let _ = env.wait_until_electrum_sees_block(Duration::from_secs(6));
73+
assert_eq!(
74+
spks.iter().cloned().collect::<BTreeSet<_>>().len(),
75+
spks.len(),
76+
"all spks must be unique",
77+
);
78+
79+
// Setup receiver.
80+
let genesis_cp = CheckPoint::new(bdk_core::BlockId {
81+
height: 0,
82+
hash: env.bitcoind.client.get_block_hash(0).unwrap(),
83+
});
84+
85+
{
86+
let electrum_client =
87+
electrum_client::Client::new(env.electrsd.electrum_url.as_str()).unwrap();
88+
let spks = spks.clone();
89+
let genesis_cp = genesis_cp.clone();
90+
c.bench_function("sync_with_electrum", move |b| {
91+
b.iter(|| {
92+
sync_with_electrum(
93+
&BdkElectrumClient::new(&electrum_client),
94+
&spks,
95+
&genesis_cp,
96+
)
97+
.expect("must not error")
98+
})
99+
});
100+
}
101+
102+
{
103+
let client = BdkElectrumClient::new(
104+
electrum_client::Client::new(env.electrsd.electrum_url.as_str()).unwrap(),
105+
);
106+
c.bench_function("sync_with_electrum_cached", move |b| {
107+
b.iter(|| sync_with_electrum(&client, &spks, &genesis_cp).expect("must not error"))
108+
});
109+
}
110+
}
111+
112+
criterion_group! {
113+
name = benches;
114+
config = Criterion::default()
115+
.sample_size(10);
116+
targets = test_sync_performance
117+
}
118+
criterion_main!(benches);

0 commit comments

Comments
 (0)