Skip to content

Commit 07090ca

Browse files
dspicherdspicher
committed
introduce proper integration tests
We introduce a new kind of integraiton test where the application performs the full flow starting from serialized blockchain data, instead of inlined bytes as was done in a test we replace here. To power this, we source in some testdata that was generated by syncing a node up to some small height. For more details, see the new README file.
1 parent 3bf0a44 commit 07090ca

File tree

13 files changed

+131
-135
lines changed

13 files changed

+131
-135
lines changed

‎Cargo.lock

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,4 +20,5 @@ tracing = { version = "0.1.37", default-features = false }
2020
tracing-subscriber = { version = "0.3.17", features = [ "env-filter", "fmt", "ansi", "tracing-log" ], default-features = false }
2121

2222
[dev-dependencies]
23+
once_cell = "1.18.0"
2324
tempfile = "3.7.0"

‎src/lib.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,15 +45,15 @@ impl std::fmt::Display for BlockHeightRange {
4545
/// Holds all available user arguments
4646
pub struct ParserOptions {
4747
// Name of the callback which gets executed for each block. (See callbacks/mod.rs)
48-
callback: Box<dyn Callback>,
48+
pub callback: Box<dyn Callback>,
4949
// Holds the relevant coin parameters we need for parsing
50-
coin: CoinType,
50+
pub coin: CoinType,
5151
// Enable this if you want to check the chain index integrity and merkle root for each block.
5252
pub verify: bool,
5353
// Path to directory where blk.dat files are stored
5454
pub blockchain_dir: std::path::PathBuf,
5555
// Range which is considered for parsing
56-
range: BlockHeightRange,
56+
pub range: BlockHeightRange,
5757
}
5858

5959
#[must_use]

‎src/parser/reader.rs

Lines changed: 0 additions & 132 deletions
Original file line numberDiff line numberDiff line change
@@ -11,135 +11,3 @@ pub trait BlockchainRead: std::io::Read {
1111
/// All types that implement `Read` get methods defined in `BlockchainRead`
1212
/// for free.
1313
impl<R: std::io::Read + ?Sized> BlockchainRead for R {}
14-
15-
#[cfg(test)]
16-
mod tests {
17-
use super::*;
18-
19-
#[test]
20-
fn test_bitcoin_parse_genesis_block() {
21-
/********** Genesis block raw data for reference (Most fields are little endian) ***********
22-
version 0x01000000 big endian??
23-
prev_hash 0x0000000000000000000000000000000000000000000000000000000000000000
24-
merkle_root 0x3ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4a
25-
timestamp 0x29ab5f49
26-
bits 0x1d00ffff
27-
nonce 0x1dac2b7c
28-
tx_count 0x01
29-
tx.version 0x01000000 big endian??
30-
tx.in_count 0x01
31-
tx.in.prev_hash 0x0000000000000000000000000000000000000000000000000000000000000000
32-
tx.in.out_id 0xffffffff
33-
tx.in.script_len 0x4d
34-
tx.in.script_sig 0x04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73
35-
tx.in.sequence 0xffffffff
36-
tx.out_count 0x01
37-
tx.out.value 0x00f2052a01000000 big endian??
38-
tx.out.script_len 0x43
39-
tx.out.script_pubkey 0x4104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac
40-
tx.lock_time 0x00000000
41-
*******************************************************************************************/
42-
let raw_data = vec![
43-
0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
44-
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
45-
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3b, 0xa3, 0xed, 0xfd, 0x7a, 0x7b,
46-
0x12, 0xb2, 0x7a, 0xc7, 0x2c, 0x3e, 0x67, 0x76, 0x8f, 0x61, 0x7f, 0xc8, 0x1b, 0xc3,
47-
0x88, 0x8a, 0x51, 0x32, 0x3a, 0x9f, 0xb8, 0xaa, 0x4b, 0x1e, 0x5e, 0x4a, 0x29, 0xab,
48-
0x5f, 0x49, 0xff, 0xff, 0x00, 0x1d, 0x1d, 0xac, 0x2b, 0x7c, 0x01, 0x01, 0x00, 0x00,
49-
0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
50-
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
51-
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x4d, 0x04, 0xff, 0xff,
52-
0x00, 0x1d, 0x01, 0x04, 0x45, 0x54, 0x68, 0x65, 0x20, 0x54, 0x69, 0x6d, 0x65, 0x73,
53-
0x20, 0x30, 0x33, 0x2f, 0x4a, 0x61, 0x6e, 0x2f, 0x32, 0x30, 0x30, 0x39, 0x20, 0x43,
54-
0x68, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x6c, 0x6f, 0x72, 0x20, 0x6f, 0x6e, 0x20, 0x62,
55-
0x72, 0x69, 0x6e, 0x6b, 0x20, 0x6f, 0x66, 0x20, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64,
56-
0x20, 0x62, 0x61, 0x69, 0x6c, 0x6f, 0x75, 0x74, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x62,
57-
0x61, 0x6e, 0x6b, 0x73, 0xff, 0xff, 0xff, 0xff, 0x01, 0x00, 0xf2, 0x05, 0x2a, 0x01,
58-
0x00, 0x00, 0x00, 0x43, 0x41, 0x04, 0x67, 0x8a, 0xfd, 0xb0, 0xfe, 0x55, 0x48, 0x27,
59-
0x19, 0x67, 0xf1, 0xa6, 0x71, 0x30, 0xb7, 0x10, 0x5c, 0xd6, 0xa8, 0x28, 0xe0, 0x39,
60-
0x09, 0xa6, 0x79, 0x62, 0xe0, 0xea, 0x1f, 0x61, 0xde, 0xb6, 0x49, 0xf6, 0xbc, 0x3f,
61-
0x4c, 0xef, 0x38, 0xc4, 0xf3, 0x55, 0x04, 0xe5, 0x1e, 0xc1, 0x12, 0xde, 0x5c, 0x38,
62-
0x4d, 0xf7, 0xba, 0x0b, 0x8d, 0x57, 0x8a, 0x4c, 0x70, 0x2b, 0x6b, 0xf1, 0x1d, 0x5f,
63-
0xac, 0x00, 0x00, 0x00, 0x00, 0xf9, 0xbe, 0xb4, 0xd9, 0xd7, 0x00, 0x00, 0x00, 0x01,
64-
0x00, 0x00, 0x00, 0x6f, 0xe2, 0x8c, 0x0a, 0xb6, 0xf1, 0xb3, 0x72, 0xc1, 0xa6, 0xa2,
65-
0x46, 0xae, 0x63, 0xf7, 0x4f, 0x93, 0x1e, 0x83, 0x65, 0xe1, 0x5a, 0x08, 0x9c, 0x68,
66-
0xd6, 0x19, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x20, 0x51, 0xfd, 0x1e, 0x4b, 0xa7,
67-
0x44, 0xbb, 0xbe, 0x68, 0x0e, 0x1f, 0xee, 0x14, 0x67, 0x7b, 0xa1, 0xa3, 0xc3, 0x54,
68-
0x0b, 0xf7, 0xb1, 0xcd, 0xb6, 0x06, 0xe8, 0x57, 0x23, 0x3e, 0x0e, 0x61, 0xbc, 0x66,
69-
0x49, 0xff, 0xff, 0x00, 0x1d, 0x01, 0xe3, 0x62, 0x99, 0x01, 0x01, 0x00, 0x00, 0x00,
70-
0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
71-
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
72-
0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x07, 0x04, 0xff, 0xff, 0x00,
73-
0x1d, 0x01, 0x04, 0xff, 0xff, 0xff, 0xff, 0x01, 0x00, 0xf2, 0x05, 0x2a, 0x01, 0x00,
74-
0x00, 0x00, 0x43, 0x41, 0x04, 0x96, 0xb5, 0x38, 0xe8, 0x53, 0x51, 0x9c, 0x72, 0x6a,
75-
0x2c, 0x91, 0xe6, 0x1e, 0xc1, 0x16, 0x00, 0xae, 0x13, 0x90, 0x81, 0x3a, 0x62, 0x7c,
76-
0x66, 0xfb, 0x8b, 0xe7, 0x94, 0x7b, 0xe6, 0x3c, 0x52, 0xda, 0x75, 0x89, 0x37, 0x95,
77-
0x15, 0xd4, 0xe0, 0xa6, 0x04, 0xf8, 0x14, 0x17, 0x81, 0xe6, 0x22, 0x94, 0x72, 0x11,
78-
0x66, 0xbf, 0x62, 0x1e, 0x73, 0xa8, 0x2c, 0xbf, 0x23, 0x42, 0xc8, 0x58, 0xee, 0xac,
79-
0x00, 0x00, 0x00, 0x0,
80-
];
81-
let inner = std::io::Cursor::new(raw_data);
82-
let mut reader = std::io::BufReader::with_capacity(200, inner);
83-
84-
// Parse block
85-
let block = reader.read_block().unwrap();
86-
87-
// Block Metadata
88-
assert_eq!(285, block.size());
89-
90-
// Block Header
91-
assert_eq!(0x0000_0001, block.header.version.to_consensus());
92-
assert_eq!(
93-
"0000000000000000000000000000000000000000000000000000000000000000",
94-
format!("{}", &block.header.prev_blockhash)
95-
);
96-
assert_eq!(
97-
"4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b",
98-
format!("{}", &block.header.merkle_root)
99-
);
100-
assert_eq!(
101-
"000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f",
102-
format!("{}", &block.header.block_hash())
103-
);
104-
105-
// Check against computed merkle root
106-
assert_eq!(
107-
&block.header.merkle_root,
108-
&block.compute_merkle_root().unwrap()
109-
);
110-
assert_eq!(1_231_006_505, block.header.time);
111-
assert_eq!(0x1d00_ffff, block.header.bits.to_consensus());
112-
assert_eq!(2_083_236_893, block.header.nonce);
113-
114-
// Tx
115-
assert_eq!(0x01, block.txdata.len());
116-
assert_eq!(0x0000_0001, block.txdata[0].version);
117-
118-
// Tx Inputs
119-
assert_eq!(0x01, block.txdata[0].input.len());
120-
assert_eq!(
121-
"0000000000000000000000000000000000000000000000000000000000000000",
122-
format!("{}", &block.txdata[0].input[0].previous_output.txid)
123-
);
124-
assert_eq!(0xffff_ffff, block.txdata[0].input[0].previous_output.vout);
125-
assert_eq!(0x4d, block.txdata[0].input[0].script_sig.len());
126-
let script = hex::decode("04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73").unwrap();
127-
assert_eq!(script, block.txdata[0].input[0].script_sig.as_bytes());
128-
assert_eq!(
129-
0xffff_ffff,
130-
block.txdata[0].input[0].sequence.to_consensus_u32()
131-
);
132-
133-
// Tx Outputs
134-
assert_eq!(0x01, block.txdata[0].output.len());
135-
assert_eq!(
136-
u64::from_be(0x00f2_052a_0100_0000),
137-
block.txdata[0].output[0].value
138-
);
139-
assert_eq!(0x43, block.txdata[0].output[0].script_pubkey.len());
140-
141-
let script = hex::decode("4104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac").unwrap();
142-
assert_eq!(script, block.txdata[0].output[0].script_pubkey.as_bytes());
143-
assert_eq!(0x0000_0000, block.txdata[0].lock_time.to_consensus_u32());
144-
}
145-
}

‎tests/mainnet.rs

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
fn storage() -> bitcoin_blockparser::parser::chain::ChainStorage {
2+
let options = bitcoin_blockparser::ParserOptions {
3+
callback: Box::new(bitcoin_blockparser::callbacks::simplestats::SimpleStats::default()),
4+
coin: "bitcoin".parse().unwrap(),
5+
verify: true,
6+
blockchain_dir: std::path::PathBuf::from("tests/testdata/mainnet"),
7+
range: bitcoin_blockparser::BlockHeightRange::new(0, Some(200)).unwrap(),
8+
};
9+
let storage = bitcoin_blockparser::parser::chain::ChainStorage::new(&options).unwrap();
10+
11+
// Discard transient diff on LevelDB files
12+
std::process::Command::new("git")
13+
.args(["checkout", "tests/testdata/mainnet"])
14+
.output()
15+
.unwrap();
16+
storage
17+
}
18+
19+
static STORAGE: once_cell::sync::Lazy<
20+
std::sync::Mutex<bitcoin_blockparser::parser::chain::ChainStorage>,
21+
> = once_cell::sync::Lazy::new(|| std::sync::Mutex::new(storage()));
22+
23+
#[test]
24+
fn test_bitcoin_genesis() {
25+
let genesis = STORAGE.lock().unwrap().get_block(0).unwrap();
26+
27+
assert_eq!(285, genesis.size());
28+
29+
// Block Header
30+
assert_eq!(0x0000_0001, genesis.header.version.to_consensus());
31+
assert_eq!(
32+
"0000000000000000000000000000000000000000000000000000000000000000",
33+
format!("{}", &genesis.header.prev_blockhash)
34+
);
35+
assert_eq!(
36+
"4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b",
37+
format!("{}", &genesis.header.merkle_root)
38+
);
39+
assert_eq!(
40+
"000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f",
41+
format!("{}", &genesis.header.block_hash())
42+
);
43+
44+
// Check against computed merkle root
45+
assert_eq!(
46+
&genesis.header.merkle_root,
47+
&genesis.compute_merkle_root().unwrap()
48+
);
49+
assert_eq!(1_231_006_505, genesis.header.time);
50+
assert_eq!(0x1d00_ffff, genesis.header.bits.to_consensus());
51+
assert_eq!(2_083_236_893, genesis.header.nonce);
52+
53+
// Tx
54+
assert_eq!(0x01, genesis.txdata.len());
55+
assert_eq!(0x0000_0001, genesis.txdata[0].version);
56+
57+
// Tx Inputs
58+
assert_eq!(0x01, genesis.txdata[0].input.len());
59+
assert_eq!(
60+
"0000000000000000000000000000000000000000000000000000000000000000",
61+
format!("{}", &genesis.txdata[0].input[0].previous_output.txid)
62+
);
63+
assert_eq!(0xffff_ffff, genesis.txdata[0].input[0].previous_output.vout);
64+
assert_eq!(0x4d, genesis.txdata[0].input[0].script_sig.len());
65+
let script = hex::decode("04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73").unwrap();
66+
assert_eq!(script, genesis.txdata[0].input[0].script_sig.as_bytes());
67+
assert_eq!(
68+
0xffff_ffff,
69+
genesis.txdata[0].input[0].sequence.to_consensus_u32()
70+
);
71+
72+
// Tx Outputs
73+
assert_eq!(0x01, genesis.txdata[0].output.len());
74+
assert_eq!(
75+
u64::from_be(0x00f2_052a_0100_0000),
76+
genesis.txdata[0].output[0].value
77+
);
78+
assert_eq!(0x43, genesis.txdata[0].output[0].script_pubkey.len());
79+
80+
let script = hex::decode("4104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac").unwrap();
81+
assert_eq!(script, genesis.txdata[0].output[0].script_pubkey.as_bytes());
82+
assert_eq!(0x0000_0000, genesis.txdata[0].lock_time.to_consensus_u32());
83+
}
84+
85+
#[test]
86+
fn test_blockdata_parsing() {
87+
for height in 0..=169 {
88+
let block = STORAGE.lock().unwrap().get_block(height).unwrap();
89+
assert_eq!(block.txdata.len(), 1);
90+
}
91+
let first_tx_block = STORAGE.lock().unwrap().get_block(170).unwrap();
92+
assert_eq!(first_tx_block.txdata.len(), 2);
93+
94+
let tx = first_tx_block.txdata.get(1).unwrap();
95+
let to_hal_finney = tx.output.get(0).unwrap();
96+
assert_eq!(to_hal_finney.value, 10 * bitcoin::Amount::ONE_BTC.to_sat());
97+
assert!(to_hal_finney.script_pubkey.is_p2pk());
98+
assert_eq!(
99+
tx.output.get(1).unwrap().value,
100+
40 * bitcoin::Amount::ONE_BTC.to_sat()
101+
);
102+
}

‎tests/testdata/README.md

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
Bitcoin Chain Prefixes
2+
======================
3+
4+
Contains Bitcoin block data as dumped by Bitcoin Core after syncing up to a small height.
5+
6+
This data can be generated roughly as follows:
7+
1. Start bitcoind on the desired network
8+
1. Sync a small number of initial blocks, then kill the daemon
9+
1. Prune LevelDB
10+
1. Remove non-block entries (the key doesn't begin with `b'b'`)
11+
1. [Copy entries over](https://github.com/wbolster/plyvel/issues/153#issuecomment-1669387180) into a new database, this results in a dramatic serialization size decrease
12+
1. Prune trailing zero bytes from `.dat` file
13+
1. `sed '$ s/\x00*$//' blk00000.dat > blk00000.dat.stripped`
14+
15+
## Mainnet
16+
17+
Synced up to height 200.

‎tests/testdata/mainnet/blk00000.dat

47.8 KB
Binary file not shown.

‎tests/testdata/mainnet/index/000003.log

28.9 KB
Binary file not shown.

‎tests/testdata/mainnet/index/CURRENT

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
MANIFEST-000002

‎tests/testdata/mainnet/index/LOCK

Whitespace-only changes.

0 commit comments

Comments
 (0)