s3 blob storage config - credentials, path style

golem-component-compilation-service/config/component-compilation-service.sample.env

@@ -63,7 +63,9 @@ GOLEM__TRACING__STDOUT__WITHOUT_TIME=false
 GOLEM__HTTP_HOST="0.0.0.0"
 GOLEM__HTTP_PORT=8084
 GOLEM__BLOB_STORAGE__TYPE="S3"
+#GOLEM__BLOB_STORAGE__CONFIG__AWS_CREDENTIALS=
 #GOLEM__BLOB_STORAGE__CONFIG__AWS_ENDPOINT_URL=
+#GOLEM__BLOB_STORAGE__CONFIG__AWS_PATH_STYLE=
 GOLEM__BLOB_STORAGE__CONFIG__COMPILATION_CACHE_BUCKET="golem-compiled-components"
 GOLEM__BLOB_STORAGE__CONFIG__COMPONENTS_BUCKET="component-store"
 GOLEM__BLOB_STORAGE__CONFIG__COMPRESSED_OPLOG_BUCKETS=["oplog-archive-1"]
@@ -73,7 +75,6 @@ GOLEM__BLOB_STORAGE__CONFIG__OBJECT_PREFIX=""
 GOLEM__BLOB_STORAGE__CONFIG__OPLOG_PAYLOAD_BUCKET="oplog-payload"
 GOLEM__BLOB_STORAGE__CONFIG__PLUGIN_WASM_FILES_BUCKET="golem-plugin-wasm-files"
 GOLEM__BLOB_STORAGE__CONFIG__REGION="us-east-1"
-GOLEM__BLOB_STORAGE__CONFIG__USE_MINIO_CREDENTIALS=false
 GOLEM__BLOB_STORAGE__CONFIG__RETRIES__MAX_ATTEMPTS=3
 GOLEM__BLOB_STORAGE__CONFIG__RETRIES__MAX_DELAY="1s"
 GOLEM__BLOB_STORAGE__CONFIG__RETRIES__MAX_JITTER_FACTOR=0.15

golem-component-compilation-service/config/component-compilation-service.toml

@@ -110,7 +110,6 @@ without_time = false
 # oplog_payload_bucket = "oplog-payload"
 # plugin_wasm_files_bucket = "golem-plugin-wasm-files"
 # region = "us-east-1"
-# use_minio_credentials = false
 # 
 # [blob_storage.config.retries]
 # max_attempts = 3

golem-service-base/src/config.rs

@@ -96,17 +96,49 @@ impl BlobStorageConfig {
     }
 }
 
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct S3BlobStorageCredentialsConfig {
+    pub access_key_id: String,
+    pub secret_access_key: String,
+    pub provider_name: String,
+}
+
+impl S3BlobStorageCredentialsConfig {
+    pub fn new(
+        access_key_id: impl Into<String>,
+        secret_access_key: impl Into<String>,
+        provider_name: impl Into<String>,
+    ) -> Self {
+        Self {
+            access_key_id: access_key_id.into(),
+            secret_access_key: secret_access_key.into(),
+            provider_name: provider_name.into(),
+        }
+    }
+}
+
+impl SafeDisplay for S3BlobStorageCredentialsConfig {
+    fn to_safe_string(&self) -> String {
+        let mut result = String::new();
+        let _ = writeln!(&mut result, "access key id: ****");
+        let _ = writeln!(&mut result, "secret access key: ****");
+        let _ = writeln!(&mut result, "provider name: {}", self.provider_name);
+        result
+    }
+}
+
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct S3BlobStorageConfig {
     pub retries: RetryConfig,
     pub region: String,
     pub object_prefix: String,
     pub aws_endpoint_url: Option<String>,
+    pub aws_credentials: Option<S3BlobStorageCredentialsConfig>,
+    pub aws_path_style: Option<bool>,
     pub compilation_cache_bucket: String,
     pub custom_data_bucket: String,
     pub oplog_payload_bucket: String,
     pub compressed_oplog_buckets: Vec<String>,
-    pub use_minio_credentials: bool,
     pub initial_component_files_bucket: String,
     pub components_bucket: String,
     pub plugin_wasm_files_bucket: String,
@@ -123,6 +155,13 @@ impl SafeDisplay for S3BlobStorageConfig {
         if let Some(endpoint_url) = &self.aws_endpoint_url {
             let _ = writeln!(&mut result, "aws_endpoint_url: {endpoint_url}");
         }
+        if let Some(aws_credentials) = &self.aws_credentials {
+            let _ = writeln!(&mut result, "aws_credentials:");
+            let _ = writeln!(&mut result, "{}", aws_credentials.to_safe_string_indented());
+        }
+        if let Some(path_style) = &self.aws_path_style {
+            let _ = writeln!(&mut result, "aws_path_style: {path_style}");
+        }
         let _ = writeln!(
             &mut result,
             "compilation cache bucket: {}",
@@ -143,11 +182,6 @@ impl SafeDisplay for S3BlobStorageConfig {
             "compressed oplog buckets: {:?}",
             self.compressed_oplog_buckets
         );
-        let _ = writeln!(
-            &mut result,
-            "use MinIO credentials: {}",
-            self.use_minio_credentials
-        );
         let _ = writeln!(
             &mut result,
             "initial component files bucket: {}",
@@ -174,8 +208,9 @@ impl Default for S3BlobStorageConfig {
             oplog_payload_bucket: "oplog-payload".to_string(),
             object_prefix: "".to_string(),
             aws_endpoint_url: None,
+            aws_credentials: None,
+            aws_path_style: None,
             compressed_oplog_buckets: vec!["oplog-archive-1".to_string()],
-            use_minio_credentials: false,
             initial_component_files_bucket: "golem-initial-component-files".to_string(),
             components_bucket: "component-store".to_string(),
             plugin_wasm_files_bucket: "golem-plugin-wasm-files".to_string(),

golem-service-base/src/storage/blob/s3.rs

@@ -58,15 +58,29 @@ impl S3BlobStorage {
             config_builder = config_builder.endpoint_url(endpoint_url);
         }
 
-        if config.use_minio_credentials {
-            let creds = Credentials::new("minioadmin", "minioadmin", None, None, "test");
+        if let Some(credentials) = config.aws_credentials.clone() {
+            let creds = Credentials::new(
+                credentials.access_key_id,
+                credentials.secret_access_key,
+                None,
+                None,
+                credentials.provider_name.leak(),
+            );
             config_builder = config_builder.credentials_provider(creds);
         }
 
         let sdk_config = config_builder.load().await;
 
+        let s3_config: aws_sdk_s3::config::Config = (&sdk_config).into();
+
+        let s3_config = if let Some(path_style) = &config.aws_path_style {
+            s3_config.to_builder().force_path_style(*path_style).build()
+        } else {
+            s3_config
+        };
+
         Self {
-            client: aws_sdk_s3::Client::new(&sdk_config),
+            client: aws_sdk_s3::Client::from_conf(s3_config),
             config,
         }
     }

golem-service-base/tests/blob_storage.rs

@@ -24,7 +24,7 @@ use futures::stream::BoxStream;
 use golem_common::model::component::ComponentId;
 use golem_common::model::environment::EnvironmentId;
 use golem_common::widen_infallible;
-use golem_service_base::config::S3BlobStorageConfig;
+use golem_service_base::config::{S3BlobStorageConfig, S3BlobStorageCredentialsConfig};
 use golem_service_base::db::sqlite::SqlitePool;
 use golem_service_base::replayable_stream::ErasedReplayableStream;
 use golem_service_base::replayable_stream::ReplayableStream;
@@ -128,7 +128,11 @@ impl GetBlobStorage for S3Test {
             region: "us-east-1".to_string(),
             object_prefix: self.prefixed.clone().unwrap_or_default(),
             aws_endpoint_url: Some(format!("http://127.0.0.1:{host_port}")),
-            use_minio_credentials: true,
+            aws_credentials: Some(S3BlobStorageCredentialsConfig::new(
+                "minioadmin",
+                "minioadmin",
+                "test",
+            )),
             ..std::default::Default::default()
         };
         create_buckets(host_port, &config).await;

golem-worker-executor/config/worker-executor.sample.env

@@ -155,7 +155,9 @@ GOLEM__ACTIVE_WORKERS__TTL="8h"
 GOLEM__AGENT_TYPES_SERVICE__TYPE="Grpc"
 GOLEM__AGENT_TYPES_SERVICE__CONFIG__CACHE_TIME_TO_IDLE="1m"
 GOLEM__BLOB_STORAGE__TYPE="S3"
+#GOLEM__BLOB_STORAGE__CONFIG__AWS_CREDENTIALS=
 #GOLEM__BLOB_STORAGE__CONFIG__AWS_ENDPOINT_URL=
+#GOLEM__BLOB_STORAGE__CONFIG__AWS_PATH_STYLE=
 GOLEM__BLOB_STORAGE__CONFIG__COMPILATION_CACHE_BUCKET="golem-compiled-components"
 GOLEM__BLOB_STORAGE__CONFIG__COMPONENTS_BUCKET="component-store"
 GOLEM__BLOB_STORAGE__CONFIG__COMPRESSED_OPLOG_BUCKETS=["oplog-archive-1"]
@@ -165,7 +167,6 @@ GOLEM__BLOB_STORAGE__CONFIG__OBJECT_PREFIX=""
 GOLEM__BLOB_STORAGE__CONFIG__OPLOG_PAYLOAD_BUCKET="oplog-payload"
 GOLEM__BLOB_STORAGE__CONFIG__PLUGIN_WASM_FILES_BUCKET="golem-plugin-wasm-files"
 GOLEM__BLOB_STORAGE__CONFIG__REGION="us-east-1"
-GOLEM__BLOB_STORAGE__CONFIG__USE_MINIO_CREDENTIALS=false
 GOLEM__BLOB_STORAGE__CONFIG__RETRIES__MAX_ATTEMPTS=3
 GOLEM__BLOB_STORAGE__CONFIG__RETRIES__MAX_DELAY="1s"
 GOLEM__BLOB_STORAGE__CONFIG__RETRIES__MAX_JITTER_FACTOR=0.15

golem-worker-executor/config/worker-executor.toml

@@ -262,7 +262,6 @@ without_time = false
 # oplog_payload_bucket = "oplog-payload"
 # plugin_wasm_files_bucket = "golem-plugin-wasm-files"
 # region = "us-east-1"
-# use_minio_credentials = false
 # 
 # [blob_storage.config.retries]
 # max_attempts = 3