Merge pull request #570 from input-output-hk/jpraynaud/542-configurable-gcp-bucket

Configurable Aggregator snapshot bucket

Author: jpraynaud

docs/root/manual/developer-docs/nodes/mithril-aggregator.md

@@ -299,6 +299,7 @@ General parameters:
 | `snapshot_directory` | `--snapshot-directory` | - | `SNAPSHOT_DIRECTORY` | Directory to store local snapshots of the **Cardano Node** | `.` | - | :heavy_check_mark: |
 | `snapshot_store_type` | - | - | `SNAPSHOT_STORE_TYPE` | Type of snapshot store to use | - | `gcp` or `local` | :heavy_check_mark: |
 | `snapshot_uploader_type` | - | - | `SNAPSHOT_UPLOADER_TYPE` | Type of snapshot uploader to use | - | `gcp` or `local` | :heavy_check_mark: |
+| `snapshot_bucket_name` | - | - | `SNAPSHOT_BUCKET_NAME` | Name of the bucket where the snapshots are stored  | - | `snapshot-bucket` | :heavy_check_mark: | Required if `snapshot_uploader_type` is `gcp`
 | `run_interval` | - | - | `RUN_INTERVAL` | Interval between two runtime cycles in ms | - | `60000` | :heavy_check_mark: |
 | `url_snapshot_manifest` | - | - | `URL_SNAPSHOT_MANIFEST` | Snapshots manifest location | - | Only if `snapshot_store_type` is `gcp`, else it should be `` | :heavy_check_mark: |
 

mithril-aggregator/config/preview.json

@@ -11,7 +11,8 @@
     "url_snapshot_manifest": "https://storage.googleapis.com/cardano-preview/snapshots.json",
     "snapshot_store_type": "local",
     "snapshot_uploader_type": "gcp",
+    "snapshot_bucket_name": "cardano-testnet",
     "stores_directory": "./mithril-aggregator/stores",
     "genesis_verification_key": "5b3132372c37332c3132342c3136312c362c3133372c3133312c3231332c3230372c3131372c3139382c38352c3137362c3139392c3136322c3234312c36382c3132332c3131392c3134352c31332c3233322c3234332c34392c3232392c322c3234392c3230352c3230352c33392c3233352c34345d",
     "limit_keys_in_stores": 5
-}
+}

mithril-aggregator/config/testnet.json

@@ -297,7 +297,7 @@ impl ServeCommand {
         debug!("SERVE command"; "config" => format!("{:?}", config));
         // Init dependencies
         let snapshot_store = config.build_snapshot_store()?;
-        let snapshot_uploader = config.build_snapshot_uploader();
+        let snapshot_uploader = config.build_snapshot_uploader()?;
 
         let sqlite_db_path = Some(config.get_sqlite_file());
 

mithril-aggregator/src/command_args.rs

@@ -49,6 +49,9 @@ pub struct Configuration {
     /// Type of snapshot uploader to use
     pub snapshot_uploader_type: SnapshotUploaderType,
 
+    /// Bucket name where the snapshots are stored if snapshot_uploader_type is Gcp
+    pub snapshot_bucket_name: Option<String>,
+
     /// Server listening IP
     pub server_ip: String,
 
@@ -109,7 +112,11 @@ impl Configuration {
     pub fn build_snapshot_store(&self) -> Result<Arc<dyn SnapshotStore>, Box<dyn Error>> {
         match self.snapshot_store_type {
             SnapshotStoreType::Gcp => Ok(Arc::new(RemoteSnapshotStore::new(
-                Box::new(GcpFileUploader::default()),
+                Box::new(GcpFileUploader::new(
+                    self.snapshot_bucket_name.to_owned().ok_or_else(|| {
+                        ConfigError::Message("missing snapshot bucket name".to_string())
+                    })?,
+                )),
                 self.url_snapshot_manifest.clone(),
             ))),
             SnapshotStoreType::Local => Ok(Arc::new(LocalSnapshotStore::new(
@@ -123,15 +130,17 @@ impl Configuration {
     }
 
     /// Create a snapshot uploader from configuration settings.
-    pub fn build_snapshot_uploader(&self) -> Arc<dyn SnapshotUploader> {
+    pub fn build_snapshot_uploader(&self) -> Result<Arc<dyn SnapshotUploader>, Box<dyn Error>> {
         match self.snapshot_uploader_type {
-            SnapshotUploaderType::Gcp => Arc::new(RemoteSnapshotUploader::new(Box::new(
-                GcpFileUploader::default(),
-            ))),
-            SnapshotUploaderType::Local => Arc::new(LocalSnapshotUploader::new(
+            SnapshotUploaderType::Gcp => Ok(Arc::new(RemoteSnapshotUploader::new(Box::new(
+                GcpFileUploader::new(self.snapshot_bucket_name.to_owned().ok_or_else(|| {
+                    ConfigError::Message("missing snapshot bucket name".to_string())
+                })?),
+            )))),
+            SnapshotUploaderType::Local => Ok(Arc::new(LocalSnapshotUploader::new(
                 self.get_server_url(),
                 &self.snapshot_directory,
-            )),
+            ))),
         }
     }
 

mithril-aggregator/src/configuration.rs

@@ -193,6 +193,7 @@ pub mod tests {
                 .to_string(),
             snapshot_store_type: SnapshotStoreType::Local,
             snapshot_uploader_type: SnapshotUploaderType::Local,
+            snapshot_bucket_name: None,
             server_ip: "0.0.0.0".to_string(),
             server_port: 8000,
             run_interval: 5000,

mithril-aggregator/src/dependency.rs

@@ -25,11 +25,10 @@ pub struct GcpFileUploader {
     bucket: String,
 }
 
-impl Default for GcpFileUploader {
-    fn default() -> Self {
-        Self {
-            bucket: "cardano-testnet".to_string(),
-        }
+impl GcpFileUploader {
+    /// GcpFileUploader factory
+    pub fn new(bucket: String) -> Self {
+        Self { bucket }
     }
 }
 

mithril-aggregator/src/tools/remote_file_uploader.rs

@@ -37,6 +37,7 @@ pub async fn initialize_dependencies(
             .to_string(),
         snapshot_store_type: SnapshotStoreType::Local,
         snapshot_uploader_type: SnapshotUploaderType::Local,
+        snapshot_bucket_name: None,
         server_ip: "0.0.0.0".to_string(),
         server_port: 8000,
         run_interval: 5000,

mithril-aggregator/tests/test_extensions/dependency.rs

@@ -65,6 +65,7 @@ services:
       - URL_SNAPSHOT_MANIFEST=https://storage.googleapis.com/cardano-${NETWORK}/snapshots.json
       - SNAPSHOT_STORE_TYPE=local
       - SNAPSHOT_UPLOADER_TYPE=gcp
+      - SNAPSHOT_BUCKET_NAME=cardano-testnet
       - DATA_STORES_DIRECTORY=/mithril-aggregator
       - STORE_RETENTION_LIMIT=5
       - CARDANO_NODE_SOCKET_PATH=/ipc/node.socket