diff --git a/python/cocoindex/sources/_engine_builtin_specs.py b/python/cocoindex/sources/_engine_builtin_specs.py index 0269c93a..dd5b3f83 100644 --- a/python/cocoindex/sources/_engine_builtin_specs.py +++ b/python/cocoindex/sources/_engine_builtin_specs.py @@ -71,6 +71,7 @@ class AmazonS3(op.SourceSpec): max_file_size: int | None = None sqs_queue_url: str | None = None redis: RedisNotification | None = None + force_path_style: bool = False class AzureBlob(op.SourceSpec): diff --git a/rust/cocoindex/src/ops/sources/amazon_s3.rs b/rust/cocoindex/src/ops/sources/amazon_s3.rs index 1cd8a93f..832ffd49 100644 --- a/rust/cocoindex/src/ops/sources/amazon_s3.rs +++ b/rust/cocoindex/src/ops/sources/amazon_s3.rs @@ -37,6 +37,7 @@ pub struct Spec { max_file_size: Option, sqs_queue_url: Option, redis: Option, + force_path_style: Option, } struct SqsContext { @@ -470,7 +471,13 @@ impl SourceFactoryBase for Factory { spec: Spec, _context: Arc, ) -> Result> { - let config = aws_config::load_defaults(BehaviorVersion::latest()).await; + let base_config = aws_config::load_defaults(BehaviorVersion::latest()).await; + + let mut s3_config_builder = aws_sdk_s3::config::Builder::from(&base_config); + if let Some(force_path_style) = spec.force_path_style { + s3_config_builder = s3_config_builder.force_path_style(force_path_style); + } + let s3_config = s3_config_builder.build(); let redis_context = if let Some(redis_config) = &spec.redis { Some(Arc::new( @@ -480,19 +487,21 @@ impl SourceFactoryBase for Factory { None }; + let sqs_context = spec.sqs_queue_url.map(|url| { + Arc::new(SqsContext { + client: aws_sdk_sqs::Client::new(&base_config), + queue_url: url, + }) + }); + Ok(Box::new(Executor { - client: Client::new(&config), + client: Client::from_conf(s3_config), bucket_name: spec.bucket_name, prefix: spec.prefix, binary: spec.binary, pattern_matcher: PatternMatcher::new(spec.included_patterns, spec.excluded_patterns)?, max_file_size: spec.max_file_size, - sqs_context: spec.sqs_queue_url.map(|url| { - Arc::new(SqsContext { - client: aws_sdk_sqs::Client::new(&config), - queue_url: url, - }) - }), + sqs_context, redis_context, })) }