diff --git a/golem-component-compilation-service/config/component-compilation-service.sample.env b/golem-component-compilation-service/config/component-compilation-service.sample.env index d53485a60b..1078d93cdc 100644 --- a/golem-component-compilation-service/config/component-compilation-service.sample.env +++ b/golem-component-compilation-service/config/component-compilation-service.sample.env @@ -63,7 +63,9 @@ GOLEM__TRACING__STDOUT__WITHOUT_TIME=false GOLEM__HTTP_HOST="0.0.0.0" GOLEM__HTTP_PORT=8084 GOLEM__BLOB_STORAGE__TYPE="S3" +#GOLEM__BLOB_STORAGE__CONFIG__AWS_CREDENTIALS= #GOLEM__BLOB_STORAGE__CONFIG__AWS_ENDPOINT_URL= +#GOLEM__BLOB_STORAGE__CONFIG__AWS_PATH_STYLE= GOLEM__BLOB_STORAGE__CONFIG__COMPILATION_CACHE_BUCKET="golem-compiled-components" GOLEM__BLOB_STORAGE__CONFIG__COMPONENTS_BUCKET="component-store" GOLEM__BLOB_STORAGE__CONFIG__COMPRESSED_OPLOG_BUCKETS=["oplog-archive-1"] @@ -73,7 +75,6 @@ GOLEM__BLOB_STORAGE__CONFIG__OBJECT_PREFIX="" GOLEM__BLOB_STORAGE__CONFIG__OPLOG_PAYLOAD_BUCKET="oplog-payload" GOLEM__BLOB_STORAGE__CONFIG__PLUGIN_WASM_FILES_BUCKET="golem-plugin-wasm-files" GOLEM__BLOB_STORAGE__CONFIG__REGION="us-east-1" -GOLEM__BLOB_STORAGE__CONFIG__USE_MINIO_CREDENTIALS=false GOLEM__BLOB_STORAGE__CONFIG__RETRIES__MAX_ATTEMPTS=3 GOLEM__BLOB_STORAGE__CONFIG__RETRIES__MAX_DELAY="1s" GOLEM__BLOB_STORAGE__CONFIG__RETRIES__MAX_JITTER_FACTOR=0.15 diff --git a/golem-component-compilation-service/config/component-compilation-service.toml b/golem-component-compilation-service/config/component-compilation-service.toml index 37656e3b1b..14f37e26d3 100644 --- a/golem-component-compilation-service/config/component-compilation-service.toml +++ b/golem-component-compilation-service/config/component-compilation-service.toml @@ -110,7 +110,6 @@ without_time = false # oplog_payload_bucket = "oplog-payload" # plugin_wasm_files_bucket = "golem-plugin-wasm-files" # region = "us-east-1" -# use_minio_credentials = false # # [blob_storage.config.retries] # max_attempts = 3 diff --git a/golem-service-base/src/config.rs b/golem-service-base/src/config.rs index 1bbde4ea0f..6f0e041af8 100644 --- a/golem-service-base/src/config.rs +++ b/golem-service-base/src/config.rs @@ -96,17 +96,49 @@ impl BlobStorageConfig { } } +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct S3BlobStorageCredentialsConfig { + pub access_key_id: String, + pub secret_access_key: String, + pub provider_name: String, +} + +impl S3BlobStorageCredentialsConfig { + pub fn new( + access_key_id: impl Into, + secret_access_key: impl Into, + provider_name: impl Into, + ) -> Self { + Self { + access_key_id: access_key_id.into(), + secret_access_key: secret_access_key.into(), + provider_name: provider_name.into(), + } + } +} + +impl SafeDisplay for S3BlobStorageCredentialsConfig { + fn to_safe_string(&self) -> String { + let mut result = String::new(); + let _ = writeln!(&mut result, "access key id: ****"); + let _ = writeln!(&mut result, "secret access key: ****"); + let _ = writeln!(&mut result, "provider name: {}", self.provider_name); + result + } +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub struct S3BlobStorageConfig { pub retries: RetryConfig, pub region: String, pub object_prefix: String, pub aws_endpoint_url: Option, + pub aws_credentials: Option, + pub aws_path_style: Option, pub compilation_cache_bucket: String, pub custom_data_bucket: String, pub oplog_payload_bucket: String, pub compressed_oplog_buckets: Vec, - pub use_minio_credentials: bool, pub initial_component_files_bucket: String, pub components_bucket: String, pub plugin_wasm_files_bucket: String, @@ -123,6 +155,13 @@ impl SafeDisplay for S3BlobStorageConfig { if let Some(endpoint_url) = &self.aws_endpoint_url { let _ = writeln!(&mut result, "aws_endpoint_url: {endpoint_url}"); } + if let Some(aws_credentials) = &self.aws_credentials { + let _ = writeln!(&mut result, "aws_credentials:"); + let _ = writeln!(&mut result, "{}", aws_credentials.to_safe_string_indented()); + } + if let Some(path_style) = &self.aws_path_style { + let _ = writeln!(&mut result, "aws_path_style: {path_style}"); + } let _ = writeln!( &mut result, "compilation cache bucket: {}", @@ -143,11 +182,6 @@ impl SafeDisplay for S3BlobStorageConfig { "compressed oplog buckets: {:?}", self.compressed_oplog_buckets ); - let _ = writeln!( - &mut result, - "use MinIO credentials: {}", - self.use_minio_credentials - ); let _ = writeln!( &mut result, "initial component files bucket: {}", @@ -174,8 +208,9 @@ impl Default for S3BlobStorageConfig { oplog_payload_bucket: "oplog-payload".to_string(), object_prefix: "".to_string(), aws_endpoint_url: None, + aws_credentials: None, + aws_path_style: None, compressed_oplog_buckets: vec!["oplog-archive-1".to_string()], - use_minio_credentials: false, initial_component_files_bucket: "golem-initial-component-files".to_string(), components_bucket: "component-store".to_string(), plugin_wasm_files_bucket: "golem-plugin-wasm-files".to_string(), diff --git a/golem-service-base/src/storage/blob/s3.rs b/golem-service-base/src/storage/blob/s3.rs index a5aa1fe4fe..ad7bd52cc2 100644 --- a/golem-service-base/src/storage/blob/s3.rs +++ b/golem-service-base/src/storage/blob/s3.rs @@ -58,15 +58,29 @@ impl S3BlobStorage { config_builder = config_builder.endpoint_url(endpoint_url); } - if config.use_minio_credentials { - let creds = Credentials::new("minioadmin", "minioadmin", None, None, "test"); + if let Some(credentials) = config.aws_credentials.clone() { + let creds = Credentials::new( + credentials.access_key_id, + credentials.secret_access_key, + None, + None, + credentials.provider_name.leak(), + ); config_builder = config_builder.credentials_provider(creds); } let sdk_config = config_builder.load().await; + let s3_config: aws_sdk_s3::config::Config = (&sdk_config).into(); + + let s3_config = if let Some(path_style) = &config.aws_path_style { + s3_config.to_builder().force_path_style(*path_style).build() + } else { + s3_config + }; + Self { - client: aws_sdk_s3::Client::new(&sdk_config), + client: aws_sdk_s3::Client::from_conf(s3_config), config, } } diff --git a/golem-service-base/tests/blob_storage.rs b/golem-service-base/tests/blob_storage.rs index 13e9d72468..4313948bde 100644 --- a/golem-service-base/tests/blob_storage.rs +++ b/golem-service-base/tests/blob_storage.rs @@ -24,7 +24,7 @@ use futures::stream::BoxStream; use golem_common::model::component::ComponentId; use golem_common::model::environment::EnvironmentId; use golem_common::widen_infallible; -use golem_service_base::config::S3BlobStorageConfig; +use golem_service_base::config::{S3BlobStorageConfig, S3BlobStorageCredentialsConfig}; use golem_service_base::db::sqlite::SqlitePool; use golem_service_base::replayable_stream::ErasedReplayableStream; use golem_service_base::replayable_stream::ReplayableStream; @@ -128,7 +128,11 @@ impl GetBlobStorage for S3Test { region: "us-east-1".to_string(), object_prefix: self.prefixed.clone().unwrap_or_default(), aws_endpoint_url: Some(format!("http://127.0.0.1:{host_port}")), - use_minio_credentials: true, + aws_credentials: Some(S3BlobStorageCredentialsConfig::new( + "minioadmin", + "minioadmin", + "test", + )), ..std::default::Default::default() }; create_buckets(host_port, &config).await; diff --git a/golem-worker-executor/config/worker-executor.sample.env b/golem-worker-executor/config/worker-executor.sample.env index d5899b4703..13a9461472 100644 --- a/golem-worker-executor/config/worker-executor.sample.env +++ b/golem-worker-executor/config/worker-executor.sample.env @@ -155,7 +155,9 @@ GOLEM__ACTIVE_WORKERS__TTL="8h" GOLEM__AGENT_TYPES_SERVICE__TYPE="Grpc" GOLEM__AGENT_TYPES_SERVICE__CONFIG__CACHE_TIME_TO_IDLE="1m" GOLEM__BLOB_STORAGE__TYPE="S3" +#GOLEM__BLOB_STORAGE__CONFIG__AWS_CREDENTIALS= #GOLEM__BLOB_STORAGE__CONFIG__AWS_ENDPOINT_URL= +#GOLEM__BLOB_STORAGE__CONFIG__AWS_PATH_STYLE= GOLEM__BLOB_STORAGE__CONFIG__COMPILATION_CACHE_BUCKET="golem-compiled-components" GOLEM__BLOB_STORAGE__CONFIG__COMPONENTS_BUCKET="component-store" GOLEM__BLOB_STORAGE__CONFIG__COMPRESSED_OPLOG_BUCKETS=["oplog-archive-1"] @@ -165,7 +167,6 @@ GOLEM__BLOB_STORAGE__CONFIG__OBJECT_PREFIX="" GOLEM__BLOB_STORAGE__CONFIG__OPLOG_PAYLOAD_BUCKET="oplog-payload" GOLEM__BLOB_STORAGE__CONFIG__PLUGIN_WASM_FILES_BUCKET="golem-plugin-wasm-files" GOLEM__BLOB_STORAGE__CONFIG__REGION="us-east-1" -GOLEM__BLOB_STORAGE__CONFIG__USE_MINIO_CREDENTIALS=false GOLEM__BLOB_STORAGE__CONFIG__RETRIES__MAX_ATTEMPTS=3 GOLEM__BLOB_STORAGE__CONFIG__RETRIES__MAX_DELAY="1s" GOLEM__BLOB_STORAGE__CONFIG__RETRIES__MAX_JITTER_FACTOR=0.15 diff --git a/golem-worker-executor/config/worker-executor.toml b/golem-worker-executor/config/worker-executor.toml index 29db8bdd02..893a5e1017 100644 --- a/golem-worker-executor/config/worker-executor.toml +++ b/golem-worker-executor/config/worker-executor.toml @@ -262,7 +262,6 @@ without_time = false # oplog_payload_bucket = "oplog-payload" # plugin_wasm_files_bucket = "golem-plugin-wasm-files" # region = "us-east-1" -# use_minio_credentials = false # # [blob_storage.config.retries] # max_attempts = 3