From db8a51572622df092b4f3815aee0891a17b1723c Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Tue, 30 Jul 2024 18:39:37 -0700 Subject: [PATCH 1/3] enhancement(aws provider): Add ability to disable request signing This can be useful when sending anonymous requests to AWS S3. Potentially it could be useful in other situations as well (e.g. sending to AWS API compatible endpoints that don't support signing). I'd really have liked to introduce this as a new "strategy" for AWS authentication configuration since it is mutually exclusive with the others, but given the way AWS authentication configuration is currently implemented as an untagged enum, adding it to the default config enum seemed like the best option. If/when we refactor this to follow https://github.com/vectordotdev/vector/blob/master/docs/specs/configuration.md#polymorphism then we can move it. Signed-off-by: Jesse Szwedko --- ...sabling_aws_request_signing.enhancement.md | 2 + src/aws/auth.rs | 64 +++++++++++++++++-- src/aws/mod.rs | 26 ++++---- src/sinks/elasticsearch/common.rs | 8 ++- src/sinks/prometheus/remote_write/service.rs | 8 ++- src/sinks/util/auth.rs | 2 +- .../sinks/base/aws_cloudwatch_logs.cue | 10 +++ .../sinks/base/aws_cloudwatch_metrics.cue | 10 +++ .../sinks/base/aws_kinesis_firehose.cue | 10 +++ .../sinks/base/aws_kinesis_streams.cue | 10 +++ .../components/sinks/base/aws_s3.cue | 10 +++ .../components/sinks/base/aws_sns.cue | 10 +++ .../components/sinks/base/aws_sqs.cue | 10 +++ .../components/sinks/base/elasticsearch.cue | 11 ++++ .../sinks/base/prometheus_remote_write.cue | 11 ++++ .../components/sources/base/aws_s3.cue | 10 +++ .../components/sources/base/aws_sqs.cue | 10 +++ 17 files changed, 198 insertions(+), 24 deletions(-) create mode 100644 changelog.d/allow_disabling_aws_request_signing.enhancement.md diff --git a/changelog.d/allow_disabling_aws_request_signing.enhancement.md b/changelog.d/allow_disabling_aws_request_signing.enhancement.md new file mode 100644 index 0000000000000..9a52ec0de27f4 --- /dev/null +++ b/changelog.d/allow_disabling_aws_request_signing.enhancement.md @@ -0,0 +1,2 @@ +AWS components can now have request signing disabled by setting `auth.sign: false` on AWS +components. This can be useful, for example, when sending anonymous requests to AWS S3. diff --git a/src/aws/auth.rs b/src/aws/auth.rs index c0e65b5dc85ba..77c3fadd5122f 100644 --- a/src/aws/auth.rs +++ b/src/aws/auth.rs @@ -175,6 +175,14 @@ pub enum AwsAuthentication { /// [aws_region]: https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints #[configurable(metadata(docs::examples = "us-west-2"))] region: Option, + + /// Whether to sign requests. + /// + /// Setting this to false can be useful when writing to an AWS S3 bucket that allows + /// anonymous requests. + #[serde(default = "crate::serde::default_true")] + #[derivative(Default(value = "true"))] + sign: bool, }, } @@ -239,7 +247,7 @@ impl AwsAuthentication { service_region: Region, proxy: &ProxyConfig, tls_options: &Option, - ) -> crate::Result { + ) -> crate::Result> { match self { Self::AccessKey { access_key_id, @@ -265,9 +273,9 @@ impl AwsAuthentication { let provider = builder.build_from_provider(provider).await; - return Ok(SharedCredentialsProvider::new(provider)); + return Ok(Some(SharedCredentialsProvider::new(provider))); } - Ok(provider) + Ok(Some(provider)) } AwsAuthentication::File { credentials_file, @@ -288,7 +296,7 @@ impl AwsAuthentication { .profile_name(profile) .configure(&provider_config) .build(); - Ok(SharedCredentialsProvider::new(profile_provider)) + Ok(Some(SharedCredentialsProvider::new(profile_provider))) } AwsAuthentication::Role { assume_role, @@ -313,9 +321,15 @@ impl AwsAuthentication { ) .await; - Ok(SharedCredentialsProvider::new(provider)) + Ok(Some(SharedCredentialsProvider::new(provider))) } - AwsAuthentication::Default { imds, region, .. } => Ok(SharedCredentialsProvider::new( + AwsAuthentication::Default { sign: false, .. } => Ok(None), + AwsAuthentication::Default { + sign: true, + imds, + region, + .. + } => Ok(Some(SharedCredentialsProvider::new( default_credentials_provider( region.clone().map(Region::new).unwrap_or(service_region), proxy, @@ -323,7 +337,7 @@ impl AwsAuthentication { *imds, ) .await?, - )), + ))), } } @@ -676,4 +690,40 @@ mod tests { _ => panic!(), } } + + #[test] + fn parsing_none() { + let config = toml::from_str::( + r#" + auth.credentials_file = "/path/to/file" + auth.profile = "foo" + "#, + ) + .unwrap(); + + match config.auth { + AwsAuthentication::File { + credentials_file, + profile, + } => { + assert_eq!(&credentials_file, "/path/to/file"); + assert_eq!(&profile, "foo"); + } + _ => panic!(), + } + + let config = toml::from_str::( + r#" + auth.no_sign = true + "#, + ) + .unwrap(); + + match config.auth { + AwsAuthentication::None { no_sign } => { + assert_eq!(&no_sign, true); + } + _ => panic!(), + } + } } diff --git a/src/aws/mod.rs b/src/aws/mod.rs index a4a68d34a2f96..f1f9ec923001b 100644 --- a/src/aws/mod.rs +++ b/src/aws/mod.rs @@ -24,7 +24,7 @@ use aws_smithy_runtime_api::client::{ runtime_components::RuntimeComponents, }; use aws_smithy_types::body::SdkBody; -use aws_types::sdk_config::SharedHttpClient; +use aws_types::sdk_config::{SharedAsyncSleep, SharedHttpClient}; use bytes::Bytes; use futures_util::FutureExt; use http::HeaderMap; @@ -201,25 +201,27 @@ pub async fn create_client_and_region( }; // Build the configuration first. - let mut config_builder = SdkConfig::builder() - .http_client(connector) - .sleep_impl(Arc::new(TokioSleep::new())) - .identity_cache(auth.credentials_cache().await?) - .credentials_provider( + let mut config_builder = SdkConfig::builder(); + + config_builder + .set_http_client(Some(SharedHttpClient::new(connector))) + .set_sleep_impl(Some(SharedAsyncSleep::new(Arc::new(TokioSleep::new())))) + .set_identity_cache(Some(auth.credentials_cache().await?)) + .set_region(Some(region.clone())) + .set_retry_config(Some(retry_config.clone())) + .set_credentials_provider( auth.credentials_provider(region.clone(), proxy, tls_options) .await?, - ) - .region(region.clone()) - .retry_config(retry_config.clone()); + ); if let Some(endpoint_override) = endpoint { - config_builder = config_builder.endpoint_url(endpoint_override); + config_builder.set_endpoint_url(Some(endpoint_override)); } if let Some(use_fips) = aws_config::default_provider::use_fips::use_fips_provider(&provider_config).await { - config_builder = config_builder.use_fips(use_fips); + config_builder.set_use_fips(Some(use_fips)); } if let Some(timeout) = timeout { @@ -234,7 +236,7 @@ pub async fn create_client_and_region( .set_connect_timeout(connect_timeout.map(Duration::from_secs)) .set_read_timeout(read_timeout.map(Duration::from_secs)); - config_builder = config_builder.timeout_config(timeout_config_builder.build()); + config_builder.set_timeout_config(Some(timeout_config_builder.build())); } let config = config_builder.build(); diff --git a/src/sinks/elasticsearch/common.rs b/src/sinks/elasticsearch/common.rs index 112a5b32f0743..b16602720c3ba 100644 --- a/src/sinks/elasticsearch/common.rs +++ b/src/sinks/elasticsearch/common.rs @@ -269,10 +269,14 @@ impl ElasticsearchCommon { #[cfg(feature = "aws-core")] pub async fn sign_request( request: &mut http::Request, - credentials_provider: &aws_credential_types::provider::SharedCredentialsProvider, + credentials_provider: &Option, region: &Option, ) -> crate::Result<()> { - crate::aws::sign_request("es", request, credentials_provider, region).await + if let Some(credentials_provider) = credentials_provider { + crate::aws::sign_request("es", request, credentials_provider, region).await + } else { + Ok(()) + } } async fn get_version( diff --git a/src/sinks/prometheus/remote_write/service.rs b/src/sinks/prometheus/remote_write/service.rs index fb459d7c4bd8e..ef1127d84fb76 100644 --- a/src/sinks/prometheus/remote_write/service.rs +++ b/src/sinks/prometheus/remote_write/service.rs @@ -94,10 +94,14 @@ impl Service for RemoteWriteService { #[cfg(feature = "aws-core")] async fn sign_request( request: &mut http::Request, - credentials_provider: &SharedCredentialsProvider, + credentials_provider: &Option, region: &Option, ) -> crate::Result<()> { - crate::aws::sign_request("aps", request, credentials_provider, region).await + if let Some(credentials_provider) = credentials_provider { + crate::aws::sign_request("aps", request, credentials_provider, region).await + } else { + Ok(()) + } } pub(super) async fn build_request( diff --git a/src/sinks/util/auth.rs b/src/sinks/util/auth.rs index 5ea99b79b7a61..a3ef231d301de 100644 --- a/src/sinks/util/auth.rs +++ b/src/sinks/util/auth.rs @@ -3,7 +3,7 @@ pub enum Auth { Basic(crate::http::Auth), #[cfg(feature = "aws-core")] Aws { - credentials_provider: aws_credential_types::provider::SharedCredentialsProvider, + credentials_provider: Option, region: aws_types::region::Region, }, } diff --git a/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue b/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue index 3c26ae7e3455c..6a8f1fc6544bb 100644 --- a/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue +++ b/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue @@ -127,6 +127,16 @@ base: components: sinks: aws_cloudwatch_logs: configuration: { required: true type: string: examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] } + sign: { + description: """ + Whether to sign requests. + + Setting this to false can be useful when writing to an AWS S3 bucket that allows + anonymous requests. + """ + required: false + type: bool: default: true + } } } batch: { diff --git a/website/cue/reference/components/sinks/base/aws_cloudwatch_metrics.cue b/website/cue/reference/components/sinks/base/aws_cloudwatch_metrics.cue index ff50dfea28990..a9df1e2c14e55 100644 --- a/website/cue/reference/components/sinks/base/aws_cloudwatch_metrics.cue +++ b/website/cue/reference/components/sinks/base/aws_cloudwatch_metrics.cue @@ -127,6 +127,16 @@ base: components: sinks: aws_cloudwatch_metrics: configuration: { required: true type: string: examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] } + sign: { + description: """ + Whether to sign requests. + + Setting this to false can be useful when writing to an AWS S3 bucket that allows + anonymous requests. + """ + required: false + type: bool: default: true + } } } batch: { diff --git a/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue b/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue index 45f999b2aa76e..21c47ceecf1be 100644 --- a/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue +++ b/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue @@ -127,6 +127,16 @@ base: components: sinks: aws_kinesis_firehose: configuration: { required: true type: string: examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] } + sign: { + description: """ + Whether to sign requests. + + Setting this to false can be useful when writing to an AWS S3 bucket that allows + anonymous requests. + """ + required: false + type: bool: default: true + } } } batch: { diff --git a/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue b/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue index 8d924f6b1a84e..e765d1de310c8 100644 --- a/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue +++ b/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue @@ -127,6 +127,16 @@ base: components: sinks: aws_kinesis_streams: configuration: { required: true type: string: examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] } + sign: { + description: """ + Whether to sign requests. + + Setting this to false can be useful when writing to an AWS S3 bucket that allows + anonymous requests. + """ + required: false + type: bool: default: true + } } } batch: { diff --git a/website/cue/reference/components/sinks/base/aws_s3.cue b/website/cue/reference/components/sinks/base/aws_s3.cue index 8d9f0b31144b3..10d522ea724e6 100644 --- a/website/cue/reference/components/sinks/base/aws_s3.cue +++ b/website/cue/reference/components/sinks/base/aws_s3.cue @@ -202,6 +202,16 @@ base: components: sinks: aws_s3: configuration: { required: true type: string: examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] } + sign: { + description: """ + Whether to sign requests. + + Setting this to false can be useful when writing to an AWS S3 bucket that allows + anonymous requests. + """ + required: false + type: bool: default: true + } } } batch: { diff --git a/website/cue/reference/components/sinks/base/aws_sns.cue b/website/cue/reference/components/sinks/base/aws_sns.cue index 4b5f15b81d68c..4999f7ec6e307 100644 --- a/website/cue/reference/components/sinks/base/aws_sns.cue +++ b/website/cue/reference/components/sinks/base/aws_sns.cue @@ -127,6 +127,16 @@ base: components: sinks: aws_sns: configuration: { required: true type: string: examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] } + sign: { + description: """ + Whether to sign requests. + + Setting this to false can be useful when writing to an AWS S3 bucket that allows + anonymous requests. + """ + required: false + type: bool: default: true + } } } encoding: { diff --git a/website/cue/reference/components/sinks/base/aws_sqs.cue b/website/cue/reference/components/sinks/base/aws_sqs.cue index 5b33a0b9f374d..6bf12faea43ec 100644 --- a/website/cue/reference/components/sinks/base/aws_sqs.cue +++ b/website/cue/reference/components/sinks/base/aws_sqs.cue @@ -127,6 +127,16 @@ base: components: sinks: aws_sqs: configuration: { required: true type: string: examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] } + sign: { + description: """ + Whether to sign requests. + + Setting this to false can be useful when writing to an AWS S3 bucket that allows + anonymous requests. + """ + required: false + type: bool: default: true + } } } encoding: { diff --git a/website/cue/reference/components/sinks/base/elasticsearch.cue b/website/cue/reference/components/sinks/base/elasticsearch.cue index 68943aa186fac..d6cb20ea0c764 100644 --- a/website/cue/reference/components/sinks/base/elasticsearch.cue +++ b/website/cue/reference/components/sinks/base/elasticsearch.cue @@ -165,6 +165,17 @@ base: components: sinks: elasticsearch: configuration: { required: true type: string: examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] } + sign: { + description: """ + Whether to sign requests. + + Setting this to false can be useful when writing to an AWS S3 bucket that allows + anonymous requests. + """ + relevant_when: "strategy = \"aws\"" + required: false + type: bool: default: true + } strategy: { description: "The authentication strategy to use." required: true diff --git a/website/cue/reference/components/sinks/base/prometheus_remote_write.cue b/website/cue/reference/components/sinks/base/prometheus_remote_write.cue index 340e12742396d..d73d16e06b3cf 100644 --- a/website/cue/reference/components/sinks/base/prometheus_remote_write.cue +++ b/website/cue/reference/components/sinks/base/prometheus_remote_write.cue @@ -142,6 +142,17 @@ base: components: sinks: prometheus_remote_write: configuration: { required: true type: string: examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] } + sign: { + description: """ + Whether to sign requests. + + Setting this to false can be useful when writing to an AWS S3 bucket that allows + anonymous requests. + """ + relevant_when: "strategy = \"aws\"" + required: false + type: bool: default: true + } strategy: { description: "The authentication strategy to use." required: true diff --git a/website/cue/reference/components/sources/base/aws_s3.cue b/website/cue/reference/components/sources/base/aws_s3.cue index 4538062a02679..6d9edc1fcd0a8 100644 --- a/website/cue/reference/components/sources/base/aws_s3.cue +++ b/website/cue/reference/components/sources/base/aws_s3.cue @@ -122,6 +122,16 @@ base: components: sources: aws_s3: configuration: { required: true type: string: examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] } + sign: { + description: """ + Whether to sign requests. + + Setting this to false can be useful when writing to an AWS S3 bucket that allows + anonymous requests. + """ + required: false + type: bool: default: true + } } } compression: { diff --git a/website/cue/reference/components/sources/base/aws_sqs.cue b/website/cue/reference/components/sources/base/aws_sqs.cue index 0d4377a71eab8..d78454fff5874 100644 --- a/website/cue/reference/components/sources/base/aws_sqs.cue +++ b/website/cue/reference/components/sources/base/aws_sqs.cue @@ -122,6 +122,16 @@ base: components: sources: aws_sqs: configuration: { required: true type: string: examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] } + sign: { + description: """ + Whether to sign requests. + + Setting this to false can be useful when writing to an AWS S3 bucket that allows + anonymous requests. + """ + required: false + type: bool: default: true + } } } client_concurrency: { From a9589602194386749e07855969e5538f5f564666 Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Tue, 30 Jul 2024 18:49:50 -0700 Subject: [PATCH 2/3] fix tests Signed-off-by: Jesse Szwedko --- src/aws/auth.rs | 27 ++++----------------------- 1 file changed, 4 insertions(+), 23 deletions(-) diff --git a/src/aws/auth.rs b/src/aws/auth.rs index 77c3fadd5122f..4180495195af0 100644 --- a/src/aws/auth.rs +++ b/src/aws/auth.rs @@ -692,36 +692,17 @@ mod tests { } #[test] - fn parsing_none() { + fn parsing_sign_false() { let config = toml::from_str::( r#" - auth.credentials_file = "/path/to/file" - auth.profile = "foo" - "#, - ) - .unwrap(); - - match config.auth { - AwsAuthentication::File { - credentials_file, - profile, - } => { - assert_eq!(&credentials_file, "/path/to/file"); - assert_eq!(&profile, "foo"); - } - _ => panic!(), - } - - let config = toml::from_str::( - r#" - auth.no_sign = true + auth.sign = false "#, ) .unwrap(); match config.auth { - AwsAuthentication::None { no_sign } => { - assert_eq!(&no_sign, true); + AwsAuthentication::Default { sign, .. } => { + assert_eq!(&no_sign, false); } _ => panic!(), } From 53a4923a2f340d4d3e01f2bc589c1ca07da19e79 Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Wed, 31 Jul 2024 09:13:44 -0700 Subject: [PATCH 3/3] Fix tests Signed-off-by: Jesse Szwedko --- src/aws/auth.rs | 4 +++- src/sinks/aws_kinesis/firehose/integration_tests.rs | 3 +++ src/sinks/elasticsearch/integration_tests.rs | 3 +++ src/sources/aws_sqs/integration_tests.rs | 1 + 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/aws/auth.rs b/src/aws/auth.rs index 4180495195af0..3f3899f4fd4cb 100644 --- a/src/aws/auth.rs +++ b/src/aws/auth.rs @@ -425,6 +425,7 @@ mod tests { load_timeout_secs: Some(10), imds: ImdsAuthentication { .. }, region: None, + sign: true, } )); } @@ -467,6 +468,7 @@ mod tests { connect_timeout: CONNECT_TIMEOUT, read_timeout: READ_TIMEOUT, }, + sign: true, } )); } @@ -702,7 +704,7 @@ mod tests { match config.auth { AwsAuthentication::Default { sign, .. } => { - assert_eq!(&no_sign, false); + assert!(!sign); } _ => panic!(), } diff --git a/src/sinks/aws_kinesis/firehose/integration_tests.rs b/src/sinks/aws_kinesis/firehose/integration_tests.rs index fc9f22e5c5a73..538a8895747b7 100644 --- a/src/sinks/aws_kinesis/firehose/integration_tests.rs +++ b/src/sinks/aws_kinesis/firehose/integration_tests.rs @@ -83,6 +83,7 @@ async fn firehose_put_records_without_partition_key() { load_timeout_secs: Some(5), imds: ImdsAuthentication::default(), region: None, + sign: true, })), endpoints: vec![elasticsearch_address()], bulk: BulkConfig { @@ -195,6 +196,7 @@ async fn firehose_put_records_with_partition_key() { load_timeout_secs: Some(5), imds: ImdsAuthentication::default(), region: None, + sign: true, })), endpoints: vec![elasticsearch_address()], bulk: BulkConfig { @@ -278,6 +280,7 @@ async fn ensure_elasticsearch_domain(domain_name: String) -> String { &None, ) .await + .unwrap() .unwrap(), ) .endpoint_url(test_region_endpoint().endpoint().unwrap()) diff --git a/src/sinks/elasticsearch/integration_tests.rs b/src/sinks/elasticsearch/integration_tests.rs index 1f627b198f8c8..53c9a088d1fa3 100644 --- a/src/sinks/elasticsearch/integration_tests.rs +++ b/src/sinks/elasticsearch/integration_tests.rs @@ -290,6 +290,7 @@ async fn auto_version_aws() { load_timeout_secs: Some(5), imds: ImdsAuthentication::default(), region: None, + sign: true, }, )), endpoints: vec![aws_server()], @@ -396,6 +397,7 @@ async fn insert_events_on_aws() { load_timeout_secs: Some(5), imds: ImdsAuthentication::default(), region: None, + sign: true, }, )), endpoints: vec![aws_server()], @@ -422,6 +424,7 @@ async fn insert_events_on_aws_with_compression() { load_timeout_secs: Some(5), imds: ImdsAuthentication::default(), region: None, + sign: true, }, )), endpoints: vec![aws_server()], diff --git a/src/sources/aws_sqs/integration_tests.rs b/src/sources/aws_sqs/integration_tests.rs index eec750b00566c..23bf85b41f0de 100644 --- a/src/sources/aws_sqs/integration_tests.rs +++ b/src/sources/aws_sqs/integration_tests.rs @@ -55,6 +55,7 @@ async fn get_sqs_client() -> aws_sdk_sqs::Client { AwsAuthentication::test_auth() .credentials_provider(Region::new("custom"), &Default::default(), &None) .await + .unwrap() .unwrap(), ) .endpoint_url(sqs_address())