tests: add testnet3 parsing tests

Also, introduce and document a small
Python script that facilitates cleaning
up the LevelDB so the amount of binary
data can be minimized.

Author: dspicher

tests/bitcoin.rs

@@ -1,24 +1,8 @@
-fn storage() -> bitcoin_blockparser::parser::chain::ChainStorage {
-    let options = bitcoin_blockparser::ParserOptions {
-        callback: Box::new(bitcoin_blockparser::callbacks::simplestats::SimpleStats::default()),
-        coin: "bitcoin".parse().unwrap(),
-        verify: true,
-        blockchain_dir: std::path::PathBuf::from("tests/testdata/bitcoin"),
-        range: bitcoin_blockparser::BlockHeightRange::new(0, Some(200)).unwrap(),
-    };
-    let storage = bitcoin_blockparser::parser::chain::ChainStorage::new(&options).unwrap();
-
-    // Discard transient diff on LevelDB files
-    std::process::Command::new("git")
-        .args(["checkout", "tests/testdata/bitcoin"])
-        .output()
-        .unwrap();
-    storage
-}
+mod common;
 
 static STORAGE: once_cell::sync::Lazy<
     std::sync::Mutex<bitcoin_blockparser::parser::chain::ChainStorage>,
-> = once_cell::sync::Lazy::new(|| std::sync::Mutex::new(storage()));
+> = once_cell::sync::Lazy::new(|| std::sync::Mutex::new(common::storage("bitcoin", 170)));
 
 #[test]
 fn test_bitcoin_genesis() {

tests/common/mod.rs

@@ -0,0 +1,17 @@
+pub fn storage(datadir: &str, max_height: u64) -> bitcoin_blockparser::parser::chain::ChainStorage {
+    let options = bitcoin_blockparser::ParserOptions {
+        callback: Box::new(bitcoin_blockparser::callbacks::simplestats::SimpleStats::default()),
+        coin: datadir.parse().unwrap(),
+        verify: true,
+        blockchain_dir: std::path::PathBuf::from(format!("tests/testdata/{datadir}")),
+        range: bitcoin_blockparser::BlockHeightRange::new(0, Some(max_height)).unwrap(),
+    };
+    let storage = bitcoin_blockparser::parser::chain::ChainStorage::new(&options).unwrap();
+
+    // Discard transient diff on LevelDB files
+    std::process::Command::new("git")
+        .args(["checkout", format!("tests/testdata/{datadir}").as_str()])
+        .output()
+        .unwrap();
+    storage
+}

tests/testdata/README.md

@@ -6,9 +6,10 @@ Contains Bitcoin block data as dumped by Bitcoin Core after syncing up to a smal
 This data can be generated roughly as follows:
  1. Start bitcoind on the desired network
  1. Sync a small number of initial blocks, then kill the daemon
- 1. Prune LevelDB
-    1. Remove non-block entries (the key doesn't begin with `b'b'`)
-    1. [Copy entries over](https://github.com/wbolster/plyvel/issues/153#issuecomment-1669387180) into a new database, this results in a dramatic serialization size decrease
+ 1. Copy the `blocks` content over to the `testdata` directory
+ 1. Prune LevelDB:
+    1. Set the correct API endpoint, max block height and path to the local DB in `clean_leveldb.py`
+    1. Run `python clean_leveldb.py` which will create a pruned DB, removing all superfluous data
  1. Prune trailing zero bytes from `.dat` file
     1. `sed '$ s/\x00*$//' blk00000.dat > blk00000.dat.stripped`
 

tests/testdata/clean_leveldb.py

@@ -0,0 +1,27 @@
+from pathlib import Path
+import requests
+import plyvel
+
+LEVEL_DB_PATH = "./testnet3/index"
+MAX_BLOCK_HEIGHT = 120
+API_ENDPOINT = "https://blockstream.info/testnet/api/"
+
+def get_blockhash(height: int) -> str:
+    return requests.get(f"{API_ENDPOINT}block-height/{height}").text.strip()
+
+def make_prefix_copy(db: plyvel.DB) -> None:
+    path = Path("./pruned-db")
+    path.mkdir()
+    db2 = plyvel.DB(str(path), create_if_missing=True, error_if_exists=True)
+    for height in range(MAX_BLOCK_HEIGHT + 1):
+        block_hash = get_blockhash(height)
+        key = b'b' + bytes.fromhex(block_hash)[::-1]
+        db2.put(key, db.get(key))
+        if height % 10 == 0:
+            print(f"processed height {height} of {MAX_BLOCK_HEIGHT}")
+    db2.close()
+
+if __name__ == "__main__":
+    db = plyvel.DB(LEVEL_DB_PATH)
+    make_prefix_copy(db)
+    db.close()

tests/testdata/testnet3/blk00000.dat

@@ -0,0 +1 @@
+MANIFEST-000002

tests/testdata/testnet3/index/000003.log

@@ -0,0 +1,23 @@
+mod common;
+
+static STORAGE: once_cell::sync::Lazy<
+    std::sync::Mutex<bitcoin_blockparser::parser::chain::ChainStorage>,
+> = once_cell::sync::Lazy::new(|| std::sync::Mutex::new(common::storage("testnet3", 120)));
+
+#[test]
+fn test_blockdata_parsing() {
+    assert_eq!(
+        STORAGE
+            .lock()
+            .unwrap()
+            .get_block(0)
+            .unwrap()
+            .block_hash()
+            .to_string(),
+        "000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943"
+    );
+    for height in 0..=120 {
+        let block = STORAGE.lock().unwrap().get_block(height).unwrap();
+        assert_eq!(block.txdata.len(), 1);
+    }
+}