Merge pull request #2475 from input-output-hk/djo/2460/reimpl_gcp_uploader

Aggregator: Re-implement the CloudBackendUploader with gcloud-storage crate

Author: Alenar

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "mithril-aggregator"
-version = "0.7.47"
+version = "0.7.48"
 description = "A Mithril Aggregator server"
 authors = { workspace = true }
 edition = { workspace = true }
@@ -22,10 +22,10 @@ anyhow = { workspace = true }
 async-trait = { workspace = true }
 chrono = { workspace = true }
 clap = { workspace = true, features = ["cargo"] }
-cloud-storage = "0.11.1"
 config = { workspace = true }
 flate2 = "1.1.1"
 gcloud-kms = { version = "1.2.0", features = ["auth"] }
+gcloud-storage = { version = "1.1.0", features = ["auth"] }
 hex = { workspace = true }
 mithril-cli-helper = { path = "../internal/mithril-cli-helper" }
 mithril-common = { path = "../mithril-common", features = ["full"] }

mithril-aggregator/Cargo.toml

@@ -11,7 +11,7 @@ use mithril_common::{
 };
 
 use crate::{
-    file_uploaders::{GcpUploader, LocalUploader},
+    file_uploaders::{CloudUploader, LocalUploader},
     services::Snapshotter,
     tools::file_archiver::FileArchive,
     DumbUploader, FileUploader,
@@ -62,7 +62,7 @@ impl AncillaryFileUploader for LocalUploader {
 }
 
 #[async_trait]
-impl AncillaryFileUploader for GcpUploader {
+impl AncillaryFileUploader for CloudUploader {
     async fn upload(
         &self,
         filepath: &Path,

mithril-aggregator/src/artifact_builder/cardano_database_artifacts/ancillary.rs

@@ -15,7 +15,7 @@ use mithril_common::{
 use slog::{error, Logger};
 
 use crate::{
-    file_uploaders::{GcpUploader, LocalUploader},
+    file_uploaders::{CloudUploader, LocalUploader},
     tools::{
         file_archiver::{appender::AppenderFile, ArchiveParameters, FileArchive, FileArchiver},
         url_sanitizer::SanitizedUrlWithTrailingSlash,
@@ -68,7 +68,7 @@ impl DigestFileUploader for LocalUploader {
 }
 
 #[async_trait]
-impl DigestFileUploader for GcpUploader {
+impl DigestFileUploader for CloudUploader {
     async fn upload(
         &self,
         filepath: &Path,

mithril-aggregator/src/artifact_builder/cardano_database_artifacts/digest.rs

@@ -12,7 +12,7 @@ use mithril_common::{
 };
 
 use crate::{
-    file_uploaders::{GcpUploader, LocalUploader},
+    file_uploaders::{CloudUploader, LocalUploader},
     services::Snapshotter,
     DumbUploader, FileUploader,
 };
@@ -102,7 +102,7 @@ impl ImmutableFilesUploader for LocalUploader {
 }
 
 #[async_trait]
-impl ImmutableFilesUploader for GcpUploader {
+impl ImmutableFilesUploader for CloudUploader {
     async fn batch_upload(
         &self,
         filepaths: &[PathBuf],
@@ -116,7 +116,7 @@ impl ImmutableFilesUploader for GcpUploader {
         let template_uri =
             MultiFilesUri::extract_template_from_uris(file_uris, immutable_file_number_extractor)?
                 .ok_or_else(|| {
-                    anyhow!("No matching template found in the uploaded files with 'GcpUploader'")
+                    anyhow!("No matching template found in the uploaded files with 'CloudUploader'")
                 })?;
 
         Ok(ImmutablesLocation::CloudStorage {

mithril-aggregator/src/artifact_builder/cardano_database_artifacts/immutable.rs

@@ -16,7 +16,7 @@ use crate::configuration::AncillaryFilesSignerConfig;
 use crate::dependency_injection::builder::SNAPSHOT_ARTIFACTS_DIR;
 use crate::dependency_injection::{DependenciesBuilder, DependenciesBuilderError, Result};
 use crate::file_uploaders::{
-    CloudRemotePath, FileUploadRetryPolicy, GcpBackendUploader, GcpUploader, LocalUploader,
+    CloudRemotePath, CloudUploader, FileUploadRetryPolicy, GCloudBackendUploader, LocalUploader,
 };
 use crate::get_dependency;
 use crate::http_server::{CARDANO_DATABASE_DOWNLOAD_PATH, SNAPSHOT_DOWNLOAD_PATH};
@@ -28,7 +28,9 @@ use crate::services::{
     SignedEntityServiceArtifactsDependencies, Snapshotter,
 };
 use crate::tools::file_archiver::FileArchiver;
+use crate::tools::DEFAULT_GCP_CREDENTIALS_JSON_ENV_VAR;
 use crate::{DumbUploader, ExecutionEnvironment, FileUploader, SnapshotUploaderType};
+
 impl DependenciesBuilder {
     async fn build_signed_entity_service(&mut self) -> Result<Arc<dyn SignedEntityService>> {
         let logger = self.root_logger();
@@ -194,7 +196,8 @@ impl DependenciesBuilder {
                     let remote_folder_path = CloudRemotePath::new("cardano-immutable-files-full");
 
                     Ok(Arc::new(
-                        self.build_gcp_uploader(remote_folder_path, allow_overwrite)?,
+                        self.build_gcp_uploader(remote_folder_path, allow_overwrite)
+                            .await?,
                     ))
                 }
                 SnapshotUploaderType::Local => {
@@ -232,11 +235,11 @@ impl DependenciesBuilder {
         get_dependency!(self.snapshot_uploader)
     }
 
-    fn build_gcp_uploader(
+    async fn build_gcp_uploader(
         &self,
         remote_folder_path: CloudRemotePath,
         allow_overwrite: bool,
-    ) -> Result<GcpUploader> {
+    ) -> Result<CloudUploader> {
         let logger = self.root_logger();
         let bucket = self
             .configuration
@@ -246,19 +249,23 @@ impl DependenciesBuilder {
                 DependenciesBuilderError::MissingConfiguration("snapshot_bucket_name".to_string())
             })?;
 
-        Ok(GcpUploader::new(
-            Arc::new(GcpBackendUploader::try_new(
-                bucket,
-                self.configuration.snapshot_use_cdn_domain(),
-                logger.clone(),
-            )?),
+        Ok(CloudUploader::new(
+            Arc::new(
+                GCloudBackendUploader::try_new(
+                    bucket,
+                    self.configuration.snapshot_use_cdn_domain(),
+                    DEFAULT_GCP_CREDENTIALS_JSON_ENV_VAR.to_string(),
+                    logger.clone(),
+                )
+                .await?,
+            ),
             remote_folder_path,
             allow_overwrite,
             FileUploadRetryPolicy::default(),
         ))
     }
 
-    fn build_cardano_database_ancillary_uploaders(
+    async fn build_cardano_database_ancillary_uploaders(
         &self,
     ) -> Result<Vec<Arc<dyn AncillaryFileUploader>>> {
         let logger = self.root_logger();
@@ -269,10 +276,10 @@ impl DependenciesBuilder {
                     let remote_folder_path =
                         CloudRemotePath::new("cardano-database").join("ancillary");
 
-                    Ok(vec![Arc::new(self.build_gcp_uploader(
-                        remote_folder_path,
-                        allow_overwrite,
-                    )?)])
+                    Ok(vec![Arc::new(
+                        self.build_gcp_uploader(remote_folder_path, allow_overwrite)
+                            .await?,
+                    )])
                 }
                 SnapshotUploaderType::Local => {
                     let server_url_prefix = self.configuration.get_server_url()?;
@@ -302,7 +309,7 @@ impl DependenciesBuilder {
         }
     }
 
-    fn build_cardano_database_immutable_uploaders(
+    async fn build_cardano_database_immutable_uploaders(
         &self,
     ) -> Result<Vec<Arc<dyn ImmutableFilesUploader>>> {
         let logger = self.root_logger();
@@ -313,10 +320,10 @@ impl DependenciesBuilder {
                     let remote_folder_path =
                         CloudRemotePath::new("cardano-database").join("immutable");
 
-                    Ok(vec![Arc::new(self.build_gcp_uploader(
-                        remote_folder_path,
-                        allow_overwrite,
-                    )?)])
+                    Ok(vec![Arc::new(
+                        self.build_gcp_uploader(remote_folder_path, allow_overwrite)
+                            .await?,
+                    )])
                 }
                 SnapshotUploaderType::Local => {
                     let server_url_prefix = self.configuration.get_server_url()?;
@@ -337,7 +344,9 @@ impl DependenciesBuilder {
         }
     }
 
-    fn build_cardano_database_digests_uploaders(&self) -> Result<Vec<Arc<dyn DigestFileUploader>>> {
+    async fn build_cardano_database_digests_uploaders(
+        &self,
+    ) -> Result<Vec<Arc<dyn DigestFileUploader>>> {
         let logger = self.root_logger();
         if self.configuration.environment() == ExecutionEnvironment::Production {
             match self.configuration.snapshot_uploader_type() {
@@ -346,10 +355,10 @@ impl DependenciesBuilder {
                     let remote_folder_path =
                         CloudRemotePath::new("cardano-database").join("digests");
 
-                    Ok(vec![Arc::new(self.build_gcp_uploader(
-                        remote_folder_path,
-                        allow_overwrite,
-                    )?)])
+                    Ok(vec![Arc::new(
+                        self.build_gcp_uploader(remote_folder_path, allow_overwrite)
+                            .await?,
+                    )])
                 }
                 SnapshotUploaderType::Local => {
                     let server_url_prefix = self.configuration.get_server_url()?;
@@ -387,15 +396,15 @@ impl DependenciesBuilder {
         let immutable_dir = self.get_cardano_db_artifacts_dir()?.join("immutable");
 
         let ancillary_builder = Arc::new(AncillaryArtifactBuilder::new(
-            self.build_cardano_database_ancillary_uploaders()?,
+            self.build_cardano_database_ancillary_uploaders().await?,
             self.get_snapshotter().await?,
             self.configuration.get_network()?,
             self.root_logger(),
         )?);
 
         let immutable_builder = Arc::new(ImmutableArtifactBuilder::new(
             immutable_dir,
-            self.build_cardano_database_immutable_uploaders()?,
+            self.build_cardano_database_immutable_uploaders().await?,
             self.get_snapshotter().await?,
             self.root_logger(),
         )?);
@@ -405,7 +414,7 @@ impl DependenciesBuilder {
 
         let digest_builder = Arc::new(DigestArtifactBuilder::new(
             self.configuration.get_server_url()?,
-            self.build_cardano_database_digests_uploaders()?,
+            self.build_cardano_database_digests_uploaders().await?,
             digests_snapshotter,
             self.configuration.get_network()?,
             digests_path,