Merge pull request #809 from input-output-hk/greg/799/refactor_sqlite_connection

refactor sqlite connection

Author: ghubertpalo

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "mithril-aggregator"
-version = "0.2.31"
+version = "0.2.32"
 description = "A Mithril Aggregator server"
 authors = { workspace = true }
 edition = { workspace = true }

mithril-aggregator/Cargo.toml

@@ -5,6 +5,7 @@ use slog_scope::{crit, debug, info};
 use sqlite::Connection;
 use std::{
     error::Error,
+    ffi::OsStr,
     fs,
     net::IpAddr,
     path::PathBuf,
@@ -34,14 +35,15 @@ use mithril_common::{
 };
 
 use crate::{
+    configuration::LIST_SNAPSHOTS_MAX_ITEMS,
     database::provider::StakePoolStore,
     event_store::{self, TransmitterService},
     http_server::routes::router,
     tools::{EraTools, GenesisTools, GenesisToolsDependency},
     AggregatorConfig, AggregatorRunner, AggregatorRuntime, CertificatePendingStore,
     CertificateStore, Configuration, DefaultConfiguration, DependencyManager, GenesisConfiguration,
-    GzipSnapshotter, MithrilSignerRegisterer, MultiSignerImpl, ProtocolParametersStore,
-    ProtocolParametersStorer, SingleSignatureStore, VerificationKeyStore,
+    GzipSnapshotter, LocalSnapshotStore, MithrilSignerRegisterer, MultiSignerImpl,
+    ProtocolParametersStore, ProtocolParametersStorer, SingleSignatureStore, VerificationKeyStore,
 };
 
 const SQLITE_MONITORING_FILE: &str = "monitoring.sqlite3";
@@ -61,6 +63,12 @@ fn setup_genesis_dependencies(
             ),
         )),
     );
+    let sqlite_connection = Arc::new(Mutex::new(Connection::open(
+        sqlite_db_path
+            .as_ref()
+            .map(|path| path.as_os_str())
+            .unwrap_or(OsStr::new(":memory:")),
+    )?));
     let immutable_file_observer = Arc::new(ImmutableFileSystemObserver::new(&config.db_directory));
     let beacon_provider = Arc::new(BeaconProviderImpl::new(
         chain_observer,
@@ -69,7 +77,7 @@ fn setup_genesis_dependencies(
     ));
     let certificate_store = Arc::new(CertificateStore::new(Box::new(SQLiteAdapter::new(
         "certificate",
-        sqlite_db_path.clone(),
+        sqlite_connection.clone(),
     )?)));
     let certificate_verifier = Arc::new(MithrilCertificateVerifier::new(slog_scope::logger()));
     let genesis_verification_key = key_decode_hex(&config.genesis_verification_key)?;
@@ -79,22 +87,20 @@ fn setup_genesis_dependencies(
     let protocol_parameters_store = Arc::new(ProtocolParametersStore::new(
         Box::new(SQLiteAdapter::new(
             "protocol_parameters",
-            sqlite_db_path.clone(),
+            sqlite_connection.clone(),
         )?),
         config.store_retention_limit,
     ));
     let verification_key_store = Arc::new(VerificationKeyStore::new(
         Box::new(SQLiteAdapter::new(
             "verification_key",
-            sqlite_db_path.clone(),
+            sqlite_connection.clone(),
         )?),
         config.store_retention_limit,
     ));
-    let stake_store = Arc::new(StakePoolStore::new(Arc::new(Mutex::new(Connection::open(
-        sqlite_db_path.clone().unwrap(),
-    )?))));
+    let stake_store = Arc::new(StakePoolStore::new(sqlite_connection.clone()));
     let single_signature_store = Arc::new(SingleSignatureStore::new(
-        Box::new(SQLiteAdapter::new("single_signature", sqlite_db_path)?),
+        Box::new(SQLiteAdapter::new("single_signature", sqlite_connection)?),
         config.store_retention_limit,
     ));
     let multi_signer = Arc::new(RwLock::new(MultiSignerImpl::new(
@@ -338,39 +344,44 @@ impl ServeCommand {
             .try_deserialize()
             .map_err(|e| format!("configuration deserialize error: {e}"))?;
         debug!("SERVE command"; "config" => format!("{config:?}"));
-        let sqlite_db_path = Some(config.get_sqlite_file());
         check_database_migration(config.get_sqlite_file())?;
 
         // Init dependencies
-        let snapshot_store = config.build_snapshot_store()?;
+        let sqlite_db_path = config.get_sqlite_file();
+        let sqlite_connection = Arc::new(Mutex::new(Connection::open(sqlite_db_path)?));
+        let snapshot_store = Arc::new(LocalSnapshotStore::new(
+            Box::new(SQLiteAdapter::new("snapshot", sqlite_connection.clone())?),
+            LIST_SNAPSHOTS_MAX_ITEMS,
+        ));
         let snapshot_uploader = config.build_snapshot_uploader()?;
 
         let certificate_pending_store = Arc::new(CertificatePendingStore::new(Box::new(
-            SQLiteAdapter::new("pending_certificate", sqlite_db_path.clone())?,
+            SQLiteAdapter::new("pending_certificate", sqlite_connection.clone())?,
         )));
         let certificate_store = Arc::new(CertificateStore::new(Box::new(SQLiteAdapter::new(
             "certificate",
-            sqlite_db_path.clone(),
+            sqlite_connection.clone(),
         )?)));
         let verification_key_store = Arc::new(VerificationKeyStore::new(
             Box::new(SQLiteAdapter::new(
                 "verification_key",
-                sqlite_db_path.clone(),
+                sqlite_connection.clone(),
             )?),
             config.store_retention_limit,
         ));
-        let stake_store = Arc::new(StakePoolStore::new(Arc::new(Mutex::new(Connection::open(
-            sqlite_db_path.clone().unwrap(),
-        )?))));
+        let stake_store = Arc::new(StakePoolStore::new(sqlite_connection.clone()));
         let single_signature_store = Arc::new(SingleSignatureStore::new(
             Box::new(SQLiteAdapter::new(
                 "single_signature",
-                sqlite_db_path.clone(),
+                sqlite_connection.clone(),
             )?),
             config.store_retention_limit,
         ));
         let protocol_parameters_store = Arc::new(ProtocolParametersStore::new(
-            Box::new(SQLiteAdapter::new("protocol_parameters", sqlite_db_path)?),
+            Box::new(SQLiteAdapter::new(
+                "protocol_parameters",
+                sqlite_connection,
+            )?),
             config.store_retention_limit,
         ));
         let chain_observer = Arc::new(

mithril-aggregator/src/command_args.rs

@@ -8,18 +8,13 @@ use std::path::PathBuf;
 use std::sync::Arc;
 
 use mithril_common::entities::{HexEncodedGenesisVerificationKey, ProtocolParameters};
-use mithril_common::store::adapter::SQLiteAdapter;
 use mithril_common::CardanoNetwork;
 
-use crate::snapshot_stores::LocalSnapshotStore;
 use crate::tools::GcpFileUploader;
-use crate::{
-    LocalSnapshotUploader, RemoteSnapshotStore, RemoteSnapshotUploader, SnapshotStore,
-    SnapshotUploader,
-};
+use crate::{LocalSnapshotUploader, RemoteSnapshotUploader, SnapshotUploader};
 
 // TODO: 'LIST_SNAPSHOTS_MAX_ITEMS' keep as const or in config, or add a parameter to `list_snapshots`?
-const LIST_SNAPSHOTS_MAX_ITEMS: usize = 20;
+pub const LIST_SNAPSHOTS_MAX_ITEMS: usize = 20;
 const SQLITE_FILE: &str = "aggregator.sqlite3";
 
 /// Aggregator configuration
@@ -46,9 +41,6 @@ pub struct Configuration {
     /// Snapshots manifest location
     pub url_snapshot_manifest: String,
 
-    /// Type of snapshot store to use
-    pub snapshot_store_type: SnapshotStoreType,
-
     /// Type of snapshot uploader to use
     pub snapshot_uploader_type: SnapshotUploaderType,
 
@@ -89,18 +81,6 @@ pub struct Configuration {
     pub era_reader_adapter_params: Option<String>,
 }
 
-/// Snapshot store type enumerates the different kinds of snapshot stores.
-/// Local storage is mainly used by development and test environements while GCP
-/// is intended for production use.
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
-#[serde(rename_all = "lowercase")]
-pub enum SnapshotStoreType {
-    /// Google storage.
-    Gcp,
-    /// Local hard drive storage.
-    Local,
-}
-
 /// Uploader needed to copy the snapshot once computed.
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 #[serde(rename_all = "lowercase")]
@@ -117,27 +97,6 @@ impl Configuration {
         format!("http://{}:{}/", self.server_ip, self.server_port)
     }
 
-    /// Create a snapshot store from the configuration settings.
-    pub fn build_snapshot_store(&self) -> Result<Arc<dyn SnapshotStore>, Box<dyn Error>> {
-        match self.snapshot_store_type {
-            SnapshotStoreType::Gcp => Ok(Arc::new(RemoteSnapshotStore::new(
-                Box::new(GcpFileUploader::new(
-                    self.snapshot_bucket_name.to_owned().ok_or_else(|| {
-                        ConfigError::Message("missing snapshot bucket name".to_string())
-                    })?,
-                )),
-                self.url_snapshot_manifest.clone(),
-            ))),
-            SnapshotStoreType::Local => Ok(Arc::new(LocalSnapshotStore::new(
-                Box::new(SQLiteAdapter::new(
-                    "snapshot",
-                    Some(self.get_sqlite_file()),
-                )?),
-                LIST_SNAPSHOTS_MAX_ITEMS,
-            ))),
-        }
-    }
-
     /// Create a snapshot uploader from configuration settings.
     pub fn build_snapshot_uploader(&self) -> Result<Arc<dyn SnapshotUploader>, Box<dyn Error>> {
         match self.snapshot_uploader_type {

mithril-aggregator/src/configuration.rs

@@ -268,7 +268,7 @@ pub mod tests {
         event_store::TransmitterService, AggregatorConfig, CertificatePendingStore,
         CertificateStore, Configuration, DependencyManager, DumbSnapshotUploader, DumbSnapshotter,
         LocalSnapshotStore, MithrilSignerRegisterer, MultiSignerImpl, ProtocolParametersStore,
-        SingleSignatureStore, SnapshotStoreType, SnapshotUploaderType, VerificationKeyStore,
+        SingleSignatureStore, SnapshotUploaderType, VerificationKeyStore,
     };
     use mithril_common::{
         certificate_chain::MithrilCertificateVerifier,
@@ -301,7 +301,6 @@ pub mod tests {
             protocol_parameters: fake_data::protocol_parameters(),
             url_snapshot_manifest: "https://storage.googleapis.com/cardano-testnet/snapshots.json"
                 .to_string(),
-            snapshot_store_type: SnapshotStoreType::Local,
             snapshot_uploader_type: SnapshotUploaderType::Local,
             snapshot_bucket_name: None,
             server_ip: "0.0.0.0".to_string(),

mithril-aggregator/src/dependency.rs

@@ -30,11 +30,10 @@ mod store;
 mod tools;
 
 pub use crate::configuration::{
-    Configuration, DefaultConfiguration, GenesisConfiguration, SnapshotStoreType,
-    SnapshotUploaderType,
+    Configuration, DefaultConfiguration, GenesisConfiguration, SnapshotUploaderType,
 };
 pub use crate::multi_signer::{MultiSigner, MultiSignerImpl, ProtocolError};
-pub use crate::snapshot_stores::{LocalSnapshotStore, RemoteSnapshotStore, SnapshotStore};
+pub use crate::snapshot_stores::{LocalSnapshotStore, SnapshotStore};
 pub use certificate_creator::{CertificateCreator, MithrilCertificateCreator};
 pub use command_args::MainOpts;
 pub use dependency::DependencyManager;

mithril-aggregator/src/lib.rs

@@ -1,9 +1,7 @@
 mod local_snapshot_store;
-mod remote_snapshot_store;
 mod snapshot_store;
 
 pub use local_snapshot_store::LocalSnapshotStore;
-pub use remote_snapshot_store::RemoteSnapshotStore;
 pub use snapshot_store::SnapshotStore;
 pub use snapshot_store::SnapshotStoreError;