switch heed

Author: Boog900

Cargo.lock

@@ -193,6 +193,7 @@ futures               = { version = "0.3", default-features = false }
 hex                   = { version = "0.4", default-features = false }
 hex-literal           = { version = "1", default-features = false }
 indexmap              = { version = "2", default-features = false }
+itertools             = { version = "0.14", default-features = false, features = ["use_std"] }
 memmap2               = { version = "0.9", default-features = false }
 monero-address        = { git = "https://github.com/monero-oxide/monero-oxide.git", rev = "7c288b058ae7f1e3d36600a2af11b3176102bcfc", default-features = false }
 monero-oxide          = { git = "https://github.com/monero-oxide/monero-oxide.git", rev = "7c288b058ae7f1e3d36600a2af11b3176102bcfc", default-features = false, features = ["std"] }

Cargo.toml

@@ -1,6 +1,7 @@
 use tower::util::MapErr;
 
-use cuprate_blockchain::{cuprate_database::RuntimeError, service::BlockchainReadHandle};
+use cuprate_database::RuntimeError;
+use cuprate_blockchain:: service::BlockchainReadHandle;
 
 /// The [`BlockchainReadHandle`] with the [`tower::Service::Error`] mapped to conform to what the consensus crate requires.
 pub type ConsensusBlockchainReadHandle =

binaries/cuprated/src/blockchain/types.rs

@@ -299,7 +299,7 @@ impl Config {
         cuprate_blockchain::config::ConfigBuilder::default()
             .network(self.network)
             .data_directory(self.fs.data_directory.clone())
-            .sync_mode(blockchain.sync_mode)
+          //  .sync_mode(blockchain.sync_mode)
             .build()
     }
 

binaries/cuprated/src/config.rs

@@ -12,13 +12,13 @@ keywords    = ["cuprate", "blockchain", "database"]
 default     = ["heed"]
 # default     = ["redb", "service"]
 # default     = ["redb-memory", "service"]
-heed        = ["cuprate-database/heed"]
-redb        = ["cuprate-database/redb"]
-redb-memory = ["cuprate-database/redb-memory"]
-serde       = ["dep:serde", "cuprate-database/serde", "cuprate-database-service/serde", "cuprate-helper/serde"]
+heed        = []
+redb        = []
+redb-memory = []
+serde       = ["dep:serde", "cuprate-database-service/serde", "cuprate-helper/serde"]
 
 [dependencies]
-cuprate-database         = { workspace = true }
+# cuprate-database         = { workspace = true }
 cuprate-database-service = { workspace = true }
 cuprate-helper           = { workspace = true, features = ["fs", "map", "crypto", "tx", "thread"] }
 cuprate-types            = { workspace = true, features = ["blockchain"] }
@@ -31,11 +31,15 @@ rand             = { workspace = true, features = ["std", "std_rng"] }
 monero-oxide     = { workspace = true, features = ["std"] }
 serde            = { workspace = true, optional = true }
 
-indexmap     = { workspace = true, features = ["rayon"] }
+heed  = { git = "https://github.com/Boog900/heed.git", branch = "wip" }
+
+itertools    = { workspace = true }
+indexmap     = { workspace = true, features = ["rayon", "std"] }
 tower        = { workspace = true }
 thread_local = { workspace = true }
 rayon        = { workspace = true }
 bytes        = { workspace = true }
+thiserror    = { workspace = true, features = ["std"] }
 
 [dev-dependencies]
 cuprate-constants  = { workspace = true }

storage/blockchain/Cargo.toml

@@ -46,7 +46,6 @@ use std::{borrow::Cow, path::PathBuf};
 #[cfg(feature = "serde")]
 use serde::{Deserialize, Serialize};
 
-use cuprate_database::{config::SyncMode, resize::ResizeAlgorithm};
 use cuprate_helper::{
     fs::{blockchain_path, CUPRATE_DATA_DIR},
     network::Network,
@@ -71,9 +70,6 @@ pub struct ConfigBuilder {
 
     data_dir: Option<PathBuf>,
 
-    /// [`Config::cuprate_database_config`].
-    db_config: cuprate_database::config::ConfigBuilder,
-
     /// [`Config::reader_threads`].
     reader_threads: Option<ReaderThreads>,
 }
@@ -87,10 +83,6 @@ impl ConfigBuilder {
         Self {
             network: Network::default(),
             data_dir: None,
-            db_config: cuprate_database::config::ConfigBuilder::new(Cow::Owned(blockchain_path(
-                &CUPRATE_DATA_DIR,
-                Network::Mainnet,
-            ))),
             reader_threads: None,
         }
     }
@@ -110,14 +102,9 @@ impl ConfigBuilder {
             .unwrap_or_else(|| CUPRATE_DATA_DIR.to_path_buf());
 
         let reader_threads = self.reader_threads.unwrap_or_default();
-        let db_config = self
-            .db_config
-            .db_directory(Cow::Owned(blockchain_path(&data_dir, self.network)))
-            .reader_threads(reader_threads.as_threads())
-            .build();
 
         Config {
-            db_config,
+            data_dir: blockchain_path(&data_dir, self.network),
             blob_data_dir: None,
             reader_threads,
         }
@@ -137,20 +124,6 @@ impl ConfigBuilder {
         self
     }
 
-    /// Calls [`cuprate_database::config::ConfigBuilder::sync_mode`].
-    #[must_use]
-    pub fn sync_mode(mut self, sync_mode: SyncMode) -> Self {
-        self.db_config = self.db_config.sync_mode(sync_mode);
-        self
-    }
-
-    /// Calls [`cuprate_database::config::ConfigBuilder::resize_algorithm`].
-    #[must_use]
-    pub fn resize_algorithm(mut self, resize_algorithm: ResizeAlgorithm) -> Self {
-        self.db_config = self.db_config.resize_algorithm(resize_algorithm);
-        self
-    }
-
     /// Set a custom [`ReaderThreads`].
     #[must_use]
     pub const fn reader_threads(mut self, reader_threads: ReaderThreads) -> Self {
@@ -164,8 +137,6 @@ impl ConfigBuilder {
     /// Good default for testing, and resource-available machines.
     #[must_use]
     pub fn fast(mut self) -> Self {
-        self.db_config = self.db_config.fast();
-
         self.reader_threads = Some(ReaderThreads::OnePerThread);
         self
     }
@@ -176,8 +147,6 @@ impl ConfigBuilder {
     /// Good default for resource-limited machines, e.g. a cheap VPS.
     #[must_use]
     pub fn low_power(mut self) -> Self {
-        self.db_config = self.db_config.low_power();
-
         self.reader_threads = Some(ReaderThreads::One);
         self
     }
@@ -187,11 +156,7 @@ impl Default for ConfigBuilder {
     fn default() -> Self {
         Self {
             network: Network::default(),
-            data_dir: Some(CUPRATE_DATA_DIR.to_path_buf()),
-            db_config: cuprate_database::config::ConfigBuilder::new(Cow::Owned(blockchain_path(
-                &CUPRATE_DATA_DIR,
-                Network::default(),
-            ))),
+            data_dir: Some(blockchain_path(&CUPRATE_DATA_DIR, Network::default())),
             reader_threads: Some(ReaderThreads::default()),
         }
     }
@@ -208,8 +173,7 @@ impl Default for ConfigBuilder {
 #[derive(Debug, Clone, PartialEq, PartialOrd)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 pub struct Config {
-    /// The database configuration.
-    pub db_config: cuprate_database::config::Config,
+    pub data_dir: PathBuf,
 
     /// The directory to store block/tx blobs.
     pub blob_data_dir: Option<PathBuf>,

storage/blockchain/src/config.rs

@@ -1,7 +1,14 @@
 use crate::config::Config;
-use cuprate_database::ConcreteEnv;
+use crate::types::{
+    AltBlockHeight, AltChainInfo, AltTransactionInfo, Amount, AmountIndices, BlockHash,
+    BlockHeight, CompactAltBlockInfo, Hash32Bytes, HeedAmountIndices, HeedUsize, KeyImage, Output,
+    PreRctOutputId, RawChainId, StorableHeed, TxHash, TxId, ZeroKey,
+};
 use cuprate_linear_tapes::{Advice, LinearTapes, Tape};
+use heed::types::U64;
+use heed::{DefaultComparator, IntegerComparator};
 use std::iter::{once, Once};
+use std::sync::OnceLock;
 
 /// The name of the ringCT outputs tape.
 pub const RCT_OUTPUTS: &str = "rct_outputs";
@@ -25,7 +32,47 @@ pub const TX_INFOS: &str = "tx_infos";
 /// The name of the block infos tape.
 pub const BLOCK_INFOS: &str = "block_infos";
 
-pub struct Database {
-    pub(crate) dynamic_tables: ConcreteEnv,
+pub static BLOCK_HEIGHTS: OnceLock<heed::Database<Hash32Bytes, HeedUsize>> = OnceLock::new();
+
+pub static KEY_IMAGES: OnceLock<heed::Database<ZeroKey, Hash32Bytes>> = OnceLock::new();
+
+pub static PRE_RCT_OUTPUTS: OnceLock<
+    heed::Database<
+        U64<heed::byteorder::NativeEndian>,
+        Output,
+        IntegerComparator,
+        IntegerComparator,
+    >,
+> = OnceLock::new();
+
+pub static TX_IDS: OnceLock<heed::Database<Hash32Bytes, HeedUsize>> = OnceLock::new();
+
+pub static TX_OUTPUTS: OnceLock<heed::Database<HeedUsize, HeedAmountIndices, IntegerComparator>> =
+    OnceLock::new();
+
+pub static ALT_CHAIN_INFOS: OnceLock<
+    heed::Database<StorableHeed<RawChainId>, StorableHeed<AltChainInfo>>,
+> = OnceLock::new();
+
+pub static ALT_BLOCK_HEIGHTS: OnceLock<heed::Database<Hash32Bytes, StorableHeed<AltBlockHeight>>> =
+    OnceLock::new();
+
+pub static ALT_BLOCKS_INFO: OnceLock<
+    heed::Database<StorableHeed<AltBlockHeight>, StorableHeed<CompactAltBlockInfo>>,
+> = OnceLock::new();
+
+pub static ALT_BLOCK_BLOBS: OnceLock<
+    heed::Database<StorableHeed<AltBlockHeight>, heed::types::Bytes>,
+> = OnceLock::new();
+
+pub static ALT_TRANSACTION_BLOBS: OnceLock<heed::Database<Hash32Bytes, heed::types::Bytes>> =
+    OnceLock::new();
+
+pub static ALT_TRANSACTION_INFOS: OnceLock<
+    heed::Database<Hash32Bytes, StorableHeed<AltTransactionInfo>>,
+> = OnceLock::new();
+
+pub struct Blockchain {
+    pub(crate) dynamic_tables: heed::Env,
     pub(crate) linear_tapes: LinearTapes,
 }

storage/blockchain/src/database.rs

@@ -0,0 +1,11 @@
+pub type DbResult<T> = Result<T, BlockchainError>;
+
+#[derive(thiserror::Error, Debug)]
+pub enum BlockchainError {
+    #[error(transparent)]
+    Heed(#[from] heed::Error),
+    #[error(transparent)]
+    IO(#[from] std::io::Error),
+    #[error("not found")]
+    NotFound,
+}