Merge #1128: feat: add bdk_sqlite crate implementing PersistBackend

evanlinjin · evanlinjin · commit 2eea0f4e906c · 2024-05-23T22:34:24.000+08:00
475c502 feat(sqlite): add bdk_sqlite crate implementing PersistBackend backed by a SQLite database (Steve Myers) b8aa76c feat(wallet): use the new `CombinedChangeSet` of `bdk_persist` (志宇) 0958ff5 feat(persist): introduce `CombinedChangeSet` (志宇) 54942a9 ci: bump build_docs rust version to nightly-2024-05-12 (Steve Myers) d975a48 docs: update README MSRV pinning to match CI (Steve Myers) Pull request description: ### Description Add "bdk_sqlite_store" crate implementing `PersistBackend` backed by a SQLite database. ### Notes to the reviewers In addition to adding a SQLite based `PersistenceBackend` this PR also: * add `CombinedChangeSet` in `bdk_persist` and update `wallet` crate to use it. * updates the `wallet/tests` to also use this new sqlite store crate. * updates `example-crates/wallet_esplora_async` to use this new sqlite store crate. * fixes out of date README instructions for MSRV. * bumps the ci `build_docs` job to rust `nightly-2024-05-12`. ### Changelog notice Changed - Update Wallet to use CombinedChangeSet for persistence. Added - Add CombinedChangeSet in bdk_persist crate. - Add bdk_sqlite crate implementing SQLite based PersistenceBackend storage for CombinedChangeSet. ### Checklists #### All Submissions: * [x] I've signed all my commits * [x] I followed the [contribution guidelines](https://github.com/bitcoindevkit/bdk/blob/master/CONTRIBUTING.md) * [x] I ran `cargo fmt` and `cargo clippy` before committing #### New Features: * [x] I've added tests for the new feature * [x] I've added docs for the new feature ACKs for top commit: evanlinjin: ACK 475c502 Tree-SHA512: 72565127994fbfff34d7359e1d36d6f13e552995bb7ea61dd38ba5329b0d15e500fa106214a83fd870f1017dc659ff1bb8fc71bc6aad0a793730ec7f23179d8c
diff --git a/.github/workflows/nightly_docs.yml b/.github/workflows/nightly_docs.yml
@@ -10,15 +10,13 @@ jobs:
       - name: Checkout sources
         uses: actions/checkout@v2
       - name: Set default toolchain
-        run: rustup default nightly-2022-12-14
+        run: rustup default nightly-2024-05-12
       - name: Set profile
         run: rustup set profile minimal
       - name: Update toolchain
         run: rustup update
       - name: Rust Cache
         uses: Swatinem/rust-cache@v2.2.1
-      - name: Pin dependencies for MSRV
-        run: cargo update -p home --precise "0.5.5"
       - name: Build docs
         run: cargo doc --no-deps
         env:
diff --git a/.gitignore b/.gitignore
@@ -7,3 +7,4 @@ Cargo.lock
 
 # Example persisted files.
 *.db
+*.sqlite*
diff --git a/Cargo.toml b/Cargo.toml
@@ -4,6 +4,7 @@ members = [
     "crates/wallet",
     "crates/chain",
     "crates/file_store",
+    "crates/sqlite",
     "crates/electrum",
     "crates/esplora",
     "crates/bitcoind_rpc",
diff --git a/README.md b/README.md
@@ -68,15 +68,9 @@ This library should compile with any combination of features with Rust 1.63.0.
 To build with the MSRV you will need to pin dependencies as follows:
 
 ```shell
-# zip 0.6.3 has MSRV 1.64.0
-cargo update -p zip --precise "0.6.2"
-# time 0.3.21 has MSRV 1.65.0
+cargo update -p zstd-sys --precise "2.0.8+zstd.1.5.5"
 cargo update -p time --precise "0.3.20"
-# jobserver 0.1.27 has MSRV 1.66.0
-cargo update -p jobserver --precise "0.1.26"
-# home 0.5.9 has MSRV 1.70.0
 cargo update -p home --precise "0.5.5"
-# proptest 1.4.0 has MSRV 1.65.0
 cargo update -p proptest --precise "1.2.0"
 ```
 
diff --git a/crates/bitcoind_rpc/Cargo.toml b/crates/bitcoind_rpc/Cargo.toml
@@ -19,7 +19,7 @@ bitcoincore-rpc = { version = "0.18" }
 bdk_chain = { path = "../chain", version = "0.14", default-features = false }
 
 [dev-dependencies]
-bdk_testenv = { path = "../testenv", default_features = false }
+bdk_testenv = { path = "../testenv", default-features = false }
 
 [features]
 default = ["std"]
diff --git a/crates/chain/src/lib.rs b/crates/chain/src/lib.rs
@@ -58,9 +58,6 @@ extern crate alloc;
 #[cfg(feature = "serde")]
 pub extern crate serde_crate as serde;
 
-#[cfg(feature = "bincode")]
-extern crate bincode;
-
 #[cfg(feature = "std")]
 #[macro_use]
 extern crate std;
diff --git a/crates/esplora/Cargo.toml b/crates/esplora/Cargo.toml
@@ -22,7 +22,7 @@ bitcoin = { version = "0.31.0", optional = true, default-features = false }
 miniscript = { version = "11.0.0", optional = true, default-features = false }
 
 [dev-dependencies]
-bdk_testenv = { path = "../testenv", default_features = false }
+bdk_testenv = { path = "../testenv", default-features = false }
 tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros"] }
 
 [features]
diff --git a/crates/file_store/README.md b/crates/file_store/README.md
@@ -1,10 +1,7 @@
 # BDK File Store
 
-This is a simple append-only flat file implementation of
-[`PersistBackend`](bdk_persist::PersistBackend).
+This is a simple append-only flat file implementation of [`PersistBackend`](bdk_persist::PersistBackend).
 
-The main structure is [`Store`](crate::Store), which can be used with [`bdk`]'s
-`Wallet` to persist wallet data into a flat file.
+The main structure is [`Store`] which works with any [`bdk_chain`] based changesets to persist data into a flat file.
 
-[`bdk`]: https://docs.rs/bdk/latest
-[`bdk_persist`]: https://docs.rs/bdk_persist/latest
+[`bdk_chain`]:https://docs.rs/bdk_chain/latest/bdk_chain/
diff --git a/crates/persist/Cargo.toml b/crates/persist/Cargo.toml
@@ -17,6 +17,6 @@ anyhow = { version = "1", default-features = false }
 bdk_chain = { path = "../chain", version = "0.14.0", default-features = false }
 
 [features]
-default = ["bdk_chain/std"]
-
-
+default = ["bdk_chain/std", "miniscript"]
+serde = ["bdk_chain/serde"]
+miniscript = ["bdk_chain/miniscript"]
diff --git a/crates/persist/README.md b/crates/persist/README.md
@@ -1,3 +1,5 @@
 # BDK Persist
 
-This crate is home to the [`PersistBackend`](crate::PersistBackend) trait which defines the behavior of a database to perform the task of persisting changes made to BDK data structures. The [`Persist`](crate::Persist) type provides a convenient wrapper around a `PersistBackend` that allows staging changes before committing them.
+This crate is home to the [`PersistBackend`] trait which defines the behavior of a database to perform the task of persisting changes made to BDK data structures. 
+
+The [`Persist`] type provides a convenient wrapper around a [`PersistBackend`] that allows staging changes before committing them.
diff --git a/crates/persist/src/changeset.rs b/crates/persist/src/changeset.rs
@@ -0,0 +1,73 @@
+#![cfg(feature = "miniscript")]
+
+use bdk_chain::{bitcoin::Network, indexed_tx_graph, keychain, local_chain, Anchor, Append};
+
+/// Changes from a combination of [`bdk_chain`] structures.
+#[derive(Debug, Clone, PartialEq)]
+#[cfg_attr(
+    feature = "serde",
+    derive(bdk_chain::serde::Deserialize, bdk_chain::serde::Serialize),
+    serde(
+        crate = "bdk_chain::serde",
+        bound(
+            deserialize = "A: Ord + bdk_chain::serde::Deserialize<'de>, K: Ord + bdk_chain::serde::Deserialize<'de>",
+            serialize = "A: Ord + bdk_chain::serde::Serialize, K: Ord + bdk_chain::serde::Serialize",
+        ),
+    )
+)]
+pub struct CombinedChangeSet<K, A> {
+    /// Changes to the [`LocalChain`](local_chain::LocalChain).
+    pub chain: local_chain::ChangeSet,
+    /// Changes to [`IndexedTxGraph`](indexed_tx_graph::IndexedTxGraph).
+    pub indexed_tx_graph: indexed_tx_graph::ChangeSet<A, keychain::ChangeSet<K>>,
+    /// Stores the network type of the transaction data.
+    pub network: Option<Network>,
+}
+
+impl<K, A> Default for CombinedChangeSet<K, A> {
+    fn default() -> Self {
+        Self {
+            chain: Default::default(),
+            indexed_tx_graph: Default::default(),
+            network: None,
+        }
+    }
+}
+
+impl<K: Ord, A: Anchor> Append for CombinedChangeSet<K, A> {
+    fn append(&mut self, other: Self) {
+        Append::append(&mut self.chain, other.chain);
+        Append::append(&mut self.indexed_tx_graph, other.indexed_tx_graph);
+        if other.network.is_some() {
+            debug_assert!(
+                self.network.is_none() || self.network == other.network,
+                "network type must either be just introduced or remain the same"
+            );
+            self.network = other.network;
+        }
+    }
+
+    fn is_empty(&self) -> bool {
+        self.chain.is_empty() && self.indexed_tx_graph.is_empty() && self.network.is_none()
+    }
+}
+
+impl<K, A> From<local_chain::ChangeSet> for CombinedChangeSet<K, A> {
+    fn from(chain: local_chain::ChangeSet) -> Self {
+        Self {
+            chain,
+            ..Default::default()
+        }
+    }
+}
+
+impl<K, A> From<indexed_tx_graph::ChangeSet<A, keychain::ChangeSet<K>>>
+    for CombinedChangeSet<K, A>
+{
+    fn from(indexed_tx_graph: indexed_tx_graph::ChangeSet<A, keychain::ChangeSet<K>>) -> Self {
+        Self {
+            indexed_tx_graph,
+            ..Default::default()
+        }
+    }
+}
diff --git a/crates/persist/src/lib.rs b/crates/persist/src/lib.rs
@@ -1,5 +1,8 @@
 #![doc = include_str!("../README.md")]
 #![no_std]
 #![warn(missing_docs)]
+
+mod changeset;
 mod persist;
+pub use changeset::*;
 pub use persist::*;
diff --git a/crates/sqlite/Cargo.toml b/crates/sqlite/Cargo.toml
@@ -0,0 +1,19 @@
+[package]
+name = "bdk_sqlite"
+version = "0.1.0"
+edition = "2021"
+license = "MIT OR Apache-2.0"
+repository = "https://github.com/bitcoindevkit/bdk"
+documentation = "https://docs.rs/bdk_sqlite"
+description = "A simple SQLite based implementation of Persist for Bitcoin Dev Kit."
+keywords = ["bitcoin", "persist", "persistence", "bdk", "sqlite"]
+authors = ["Bitcoin Dev Kit Developers"]
+readme = "README.md"
+
+[dependencies]
+anyhow = { version = "1", default-features = false }
+bdk_chain = { path = "../chain", version = "0.14.0", features = ["serde", "miniscript"] }
+bdk_persist = { path = "../persist", version = "0.2.0", features = ["serde"] }
+rusqlite = { version = "0.31.0", features = ["bundled"] }
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
diff --git a/crates/sqlite/README.md b/crates/sqlite/README.md
@@ -0,0 +1,8 @@
+# BDK SQLite
+
+This is a simple [SQLite] relational database schema backed implementation of [`PersistBackend`](bdk_persist::PersistBackend).
+
+The main structure is `Store` which persists [`bdk_persist`] `CombinedChangeSet` data into a SQLite database file.
+
+[`bdk_persist`]:https://docs.rs/bdk_persist/latest/bdk_persist/
+[SQLite]: https://www.sqlite.org/index.html
diff --git a/crates/sqlite/schema/schema_0.sql b/crates/sqlite/schema/schema_0.sql
@@ -0,0 +1,69 @@
+-- schema version control
+CREATE TABLE version
+(
+    version INTEGER
+) STRICT;
+INSERT INTO version
+VALUES (1);
+
+-- network is the valid network for all other table data
+CREATE TABLE network
+(
+    name TEXT UNIQUE NOT NULL
+) STRICT;
+
+-- keychain is the json serialized keychain structure as JSONB,
+-- descriptor is the complete descriptor string,
+-- descriptor_id is a sha256::Hash id of the descriptor string w/o the checksum,
+-- last revealed index is a u32
+CREATE TABLE keychain
+(
+    keychain      BLOB PRIMARY KEY NOT NULL,
+    descriptor    TEXT             NOT NULL,
+    descriptor_id BLOB             NOT NULL,
+    last_revealed INTEGER
+) STRICT;
+
+-- hash is block hash hex string,
+-- block height is a u32,
+CREATE TABLE block
+(
+    hash   TEXT PRIMARY KEY NOT NULL,
+    height INTEGER          NOT NULL
+) STRICT;
+
+-- txid is transaction hash hex string (reversed)
+-- whole_tx is a consensus encoded transaction,
+-- last seen is a u64 unix epoch seconds
+CREATE TABLE tx
+(
+    txid      TEXT PRIMARY KEY NOT NULL,
+    whole_tx  BLOB,
+    last_seen INTEGER
+) STRICT;
+
+-- Outpoint txid hash hex string (reversed)
+-- Outpoint vout
+-- TxOut value as SATs
+-- TxOut script consensus encoded
+CREATE TABLE txout
+(
+    txid   TEXT    NOT NULL,
+    vout   INTEGER NOT NULL,
+    value  INTEGER NOT NULL,
+    script BLOB    NOT NULL,
+    PRIMARY KEY (txid, vout)
+) STRICT;
+
+-- join table between anchor and tx
+-- block hash hex string
+-- anchor is a json serialized Anchor structure as JSONB,
+-- txid is transaction hash hex string (reversed)
+CREATE TABLE anchor_tx
+(
+    block_hash          TEXT NOT NULL,
+    anchor              BLOB NOT NULL,
+    txid                TEXT NOT NULL REFERENCES tx (txid),
+    UNIQUE (anchor, txid),
+    FOREIGN KEY (block_hash) REFERENCES block(hash)
+) STRICT;
diff --git a/crates/sqlite/src/lib.rs b/crates/sqlite/src/lib.rs
@@ -0,0 +1,34 @@
+#![doc = include_str!("../README.md")]
+// only enables the `doc_cfg` feature when the `docsrs` configuration attribute is defined
+#![cfg_attr(docsrs, feature(doc_cfg))]
+
+mod schema;
+mod store;
+
+use bdk_chain::bitcoin::Network;
+pub use rusqlite;
+pub use store::Store;
+
+/// Error that occurs while reading or writing change sets with the SQLite database.
+#[derive(Debug)]
+pub enum Error {
+    /// Invalid network, cannot change the one already stored in the database.
+    Network { expected: Network, given: Network },
+    /// SQLite error.
+    Sqlite(rusqlite::Error),
+}
+
+impl core::fmt::Display for Error {
+    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+        match self {
+            Self::Network { expected, given } => write!(
+                f,
+                "network error trying to read or write change set, expected {}, given {}",
+                expected, given
+            ),
+            Self::Sqlite(e) => write!(f, "sqlite error reading or writing changeset: {}", e),
+        }
+    }
+}
+
+impl std::error::Error for Error {}
diff --git a/crates/sqlite/src/schema.rs b/crates/sqlite/src/schema.rs
diff --git a/crates/sqlite/src/store.rs b/crates/sqlite/src/store.rs
diff --git a/crates/wallet/Cargo.toml b/crates/wallet/Cargo.toml
diff --git a/crates/wallet/src/wallet/mod.rs b/crates/wallet/src/wallet/mod.rs
diff --git a/crates/wallet/tests/wallet.rs b/crates/wallet/tests/wallet.rs
diff --git a/example-crates/wallet_esplora_async/Cargo.toml b/example-crates/wallet_esplora_async/Cargo.toml
diff --git a/example-crates/wallet_esplora_async/src/main.rs b/example-crates/wallet_esplora_async/src/main.rs
diff --git a/nursery/coin_select/src/bnb.rs b/nursery/coin_select/src/bnb.rs

Original file line number	Diff line number	Diff line change
`@@ -7,3 +7,4 @@ Cargo.lock`
`7`	`7`
`8`	`8`	`# Example persisted files.`
`9`	`9`	`*.db`
	`10`	`+.sqlite`