Add new MRAP test to S3 canary (#3109)

This PR also replaces the `2023_09_25` canary with current release
*(`2023_10_26`)*

----

_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._

Author: Zelda Hessler

tools/ci-cdk/canary-lambda/README.md

@@ -34,6 +34,7 @@ invoke the canary:
 
 ```bash
 export CANARY_S3_BUCKET_NAME=<your bucket name>
+export CANARY_S3_MRAP_BUCKET_ARN=<your MRAP bucket ARN>
 # run with `--all-features` so you run all canaries (including canaries that don't work against older versions)
 cargo run --all-features -- --local
 ```

tools/ci-cdk/canary-lambda/src/canary.rs

@@ -50,6 +50,7 @@ pub fn get_canaries_to_run(
 
 pub struct CanaryEnv {
     pub(crate) s3_bucket_name: String,
+    pub(crate) s3_mrap_bucket_arn: String,
     pub(crate) expected_transcribe_result: String,
     #[allow(dead_code)]
     pub(crate) page_size: usize,
@@ -59,6 +60,7 @@ impl fmt::Debug for CanaryEnv {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.debug_struct("CanaryEnv")
             .field("s3_bucket_name", &"*** redacted ***")
+            .field("s3_mrap_bucket_arn", &"*** redacted ***")
             .field(
                 "expected_transcribe_result",
                 &self.expected_transcribe_result,
@@ -72,6 +74,9 @@ impl CanaryEnv {
         // S3 bucket name to test against
         let s3_bucket_name =
             env::var("CANARY_S3_BUCKET_NAME").expect("CANARY_S3_BUCKET_NAME must be set");
+        // S3 MRAP bucket name to test against
+        let s3_mrap_bucket_arn =
+            env::var("CANARY_S3_MRAP_BUCKET_ARN").expect("CANARY_S3_MRAP_BUCKET_ARN must be set");
 
         // Expected transcription from Amazon Transcribe from the embedded audio file.
         // This is an environment variable so that the code doesn't need to be changed if
@@ -89,6 +94,7 @@ impl CanaryEnv {
 
         Self {
             s3_bucket_name,
+            s3_mrap_bucket_arn,
             expected_transcribe_result,
             page_size,
         }

tools/ci-cdk/canary-lambda/src/latest/paginator_canary.rs

@@ -38,7 +38,7 @@ pub async fn paginator_canary(client: ec2::Client, page_size: usize) -> anyhow::
         }
         num_pages += 1;
     }
-    if dbg!(num_pages) < 2 {
+    if num_pages < 2 {
         bail!(
             "expected 3+ pages containing ~60 results but got {} pages",
             num_pages
@@ -59,7 +59,7 @@ pub async fn paginator_canary(client: ec2::Client, page_size: usize) -> anyhow::
 
 #[cfg(test)]
 mod test {
-    use crate::latest::paginator_canary::paginator_canary;
+    use super::paginator_canary;
 
     #[tokio::test]
     async fn test_paginator() {

tools/ci-cdk/canary-lambda/src/latest/s3_canary.rs

@@ -8,6 +8,7 @@ use crate::{mk_canary, CanaryEnv};
 use anyhow::Context;
 use aws_config::SdkConfig;
 use aws_sdk_s3 as s3;
+use s3::config::Region;
 use s3::presigning::PresigningConfig;
 use s3::primitives::ByteStream;
 use std::time::Duration;
@@ -17,10 +18,15 @@ const METADATA_TEST_VALUE: &str = "some   value";
 
 mk_canary!("s3", |sdk_config: &SdkConfig, env: &CanaryEnv| s3_canary(
     s3::Client::new(sdk_config),
-    env.s3_bucket_name.clone()
+    env.s3_bucket_name.clone(),
+    env.s3_mrap_bucket_arn.clone()
 ));
 
-pub async fn s3_canary(client: s3::Client, s3_bucket_name: String) -> anyhow::Result<()> {
+pub async fn s3_canary(
+    client: s3::Client,
+    s3_bucket_name: String,
+    s3_mrap_bucket_arn: String,
+) -> anyhow::Result<()> {
     let test_key = Uuid::new_v4().as_u128().to_string();
 
     // Look for the test object and expect that it doesn't exist
@@ -82,39 +88,104 @@ pub async fn s3_canary(client: s3::Client, s3_bucket_name: String) -> anyhow::Re
         return Err(CanaryError(format!("presigned URL returned bad data: {:?}", response)).into());
     }
 
-    let mut result = Ok(());
-    match output.metadata() {
-        Some(map) => {
-            // Option::as_deref doesn't work here since the deref of &String is String
-            let value = map.get("something").map(|s| s.as_str()).unwrap_or("");
-            if value != METADATA_TEST_VALUE {
-                result = Err(CanaryError(format!(
+    let metadata_value = output
+        .metadata()
+        .and_then(|m| m.get("something"))
+        .map(String::as_str);
+    let result: anyhow::Result<()> = match metadata_value {
+        Some(value) => {
+            if value == METADATA_TEST_VALUE {
+                let payload = output
+                    .body
+                    .collect()
+                    .await
+                    .context("download s3::GetObject[2] body")?
+                    .into_bytes();
+                if std::str::from_utf8(payload.as_ref()).context("s3 payload")? == "test" {
+                    Ok(())
+                } else {
+                    Err(CanaryError("S3 object body didn't match what was put there".into()).into())
+                }
+            } else {
+                Err(CanaryError(format!(
                     "S3 metadata was incorrect. Expected `{}` but got `{}`.",
                     METADATA_TEST_VALUE, value
                 ))
-                .into());
+                .into())
             }
         }
-        None => {
-            result = Err(CanaryError("S3 metadata was missing".into()).into());
-        }
-    }
+        None => Err(CanaryError("S3 metadata was missing".into()).into()),
+    };
 
-    let payload = output
-        .body
-        .collect()
+    // Delete the test object
+    client
+        .delete_object()
+        .bucket(&s3_bucket_name)
+        .key(&test_key)
+        .send()
         .await
-        .context("download s3::GetObject[2] body")?
-        .into_bytes();
-    if std::str::from_utf8(payload.as_ref()).context("s3 payload")? != "test" {
-        result = Err(CanaryError("S3 object body didn't match what was put there".into()).into());
-    }
+        .context("s3::DeleteObject")?;
+
+    // Return early if the result is an error
+    result?;
+
+    // We deliberately use a region that doesn't exist here so that we can
+    // ensure these requests are SigV4a requests. Because the current endpoint
+    // resolver always resolves the wildcard region ('*') for SigV4a requests,
+    // setting a fictitious region ensures that the request would fail if it was
+    // a SigV4 request. Therefore, because the request doesn't fail, we can be
+    // sure it's a successful Sigv4a request.
+    let config_override = s3::Config::builder().region(Region::new("parts-unknown"));
+    // Put the test object
+    client
+        .put_object()
+        .bucket(&s3_mrap_bucket_arn)
+        .key(&test_key)
+        .body(ByteStream::from_static(b"test"))
+        .metadata("something", METADATA_TEST_VALUE)
+        .customize()
+        .config_override(config_override.clone())
+        .send()
+        .await
+        .context("s3::PutObject[MRAP]")?;
+
+    // Get the test object and verify it looks correct
+    let output = client
+        .get_object()
+        .bucket(&s3_mrap_bucket_arn)
+        .key(&test_key)
+        .customize()
+        .config_override(config_override.clone())
+        .send()
+        .await
+        .context("s3::GetObject[MRAP]")?;
+
+    let metadata_value = output
+        .metadata()
+        .and_then(|m| m.get("something"))
+        .map(String::as_str);
+    let result = match metadata_value {
+        Some(value) => {
+            if value == METADATA_TEST_VALUE {
+                Ok(())
+            } else {
+                Err(CanaryError(format!(
+                    "S3 metadata was incorrect. Expected `{}` but got `{}`.",
+                    METADATA_TEST_VALUE, value
+                ))
+                .into())
+            }
+        }
+        None => Err(CanaryError("S3 metadata was missing".into()).into()),
+    };
 
     // Delete the test object
     client
         .delete_object()
-        .bucket(&s3_bucket_name)
+        .bucket(&s3_mrap_bucket_arn)
         .key(&test_key)
+        .customize()
+        .config_override(config_override)
         .send()
         .await
         .context("s3::DeleteObject")?;
@@ -123,8 +194,12 @@ pub async fn s3_canary(client: s3::Client, s3_bucket_name: String) -> anyhow::Re
 }
 
 // This test runs against an actual AWS account. Comment out the `ignore` to run it.
-// Be sure to set the `TEST_S3_BUCKET` environment variable to the S3 bucket to use,
-// and also make sure the credential profile sets the region (or set `AWS_DEFAULT_PROFILE`).
+// Be sure the following environment variables are set:
+//
+// - `TEST_S3_BUCKET`: The S3 bucket to use
+// - `TEST_S3_MRAP_BUCKET_ARN`: The MRAP bucket ARN to use
+//
+// Also, make sure the correct region (likely `us-west-2`) by the credentials or explictly.
 #[ignore]
 #[cfg(test)]
 #[tokio::test]
@@ -134,6 +209,7 @@ async fn test_s3_canary() {
     s3_canary(
         client,
         std::env::var("TEST_S3_BUCKET").expect("TEST_S3_BUCKET must be set"),
+        std::env::var("TEST_S3_MRAP_BUCKET_ARN").expect("TEST_S3_MRAP_BUCKET_ARN must be set"),
     )
     .await
     .expect("success");

tools/ci-cdk/canary-lambda/src/main.rs

@@ -26,11 +26,11 @@ mod latest;
 #[cfg(feature = "latest")]
 pub(crate) use latest as current_canary;
 
-// NOTE: This module can be deleted 3 releases after release-2023-09-25
-#[cfg(feature = "release-2023-09-25")]
-mod release_2023_09_25;
-#[cfg(feature = "release-2023-09-25")]
-pub(crate) use release_2023_09_25 as current_canary;
+// NOTE: This module can be deleted 3 releases after release-2023-10-26
+#[cfg(feature = "release-2023-10-26")]
+mod release_2023_10_26;
+#[cfg(feature = "release-2023-10-26")]
+pub(crate) use release_2023_10_26 as current_canary;
 
 #[tokio::main]
 async fn main() -> Result<(), Error> {

tools/ci-cdk/canary-lambda/src/release_2023_09_25.rs renamed to tools/ci-cdk/canary-lambda/src/release_2023_10_26.rs

@@ -10,7 +10,6 @@ use aws_sdk_ec2 as ec2;
 use aws_sdk_ec2::types::InstanceType;
 
 use crate::CanaryEnv;
-use tokio_stream::StreamExt;
 
 mk_canary!(
     "ec2_paginator",
@@ -39,7 +38,7 @@ pub async fn paginator_canary(client: ec2::Client, page_size: usize) -> anyhow::
         }
         num_pages += 1;
     }
-    if dbg!(num_pages) < 2 {
+    if num_pages < 2 {
         bail!(
             "expected 3+ pages containing ~60 results but got {} pages",
             num_pages
@@ -60,7 +59,7 @@ pub async fn paginator_canary(client: ec2::Client, page_size: usize) -> anyhow::
 
 #[cfg(test)]
 mod test {
-    use crate::current_canary::paginator_canary::paginator_canary;
+    use super::paginator_canary;
 
     #[tokio::test]
     async fn test_paginator() {

tools/ci-cdk/canary-lambda/src/release_2023_09_25/paginator_canary.rs renamed to tools/ci-cdk/canary-lambda/src/release_2023_10_26/paginator_canary.rs

@@ -66,7 +66,7 @@ const REQUIRED_SDK_CRATES: &[&str] = &[
 // The elements in this `Vec` should be sorted in an ascending order by the release date.
 lazy_static! {
     static ref NOTABLE_SDK_RELEASE_TAGS: Vec<ReleaseTag> = vec![
-        ReleaseTag::from_str("release-2023-09-25").unwrap(), // last version before `Stream` trait removal
+        ReleaseTag::from_str("release-2023-10-26").unwrap(), // last version before addition of Sigv4a MRAP test
     ];
 }
 
@@ -442,7 +442,7 @@ aws-sdk-transcribestreaming = { path = "some/sdk/path/transcribestreaming" }
 
 [features]
 latest = []
-"release-2023-09-25" = []
+"release-2023-10-26" = []
 default = ["latest"]
 "#,
             generate_crate_manifest(CrateSource::Path("some/sdk/path".into())).expect("success")
@@ -506,7 +506,7 @@ aws-sdk-transcribestreaming = "0.16.0"
 
 [features]
 latest = []
-"release-2023-09-25" = []
+"release-2023-10-26" = []
 default = ["latest"]
 "#,
             generate_crate_manifest(CrateSource::VersionsManifest {
@@ -585,14 +585,14 @@ default = ["latest"]
             }),
         );
         assert_eq!(
-            "release-2023-09-25".to_string(),
+            "release-2023-10-26".to_string(),
             enabled_feature(&CrateSource::VersionsManifest {
                 versions: versions.clone(),
-                release_tag: "release-2023-09-25".parse().unwrap(),
+                release_tag: "release-2023-10-26".parse().unwrap(),
             }),
         );
         assert_eq!(
-            "release-2023-09-25".to_string(),
+            "release-2023-10-26".to_string(),
             enabled_feature(&CrateSource::VersionsManifest {
                 versions,
                 release_tag: "release-2023-01-13".parse().unwrap(),