Update to latest SDKs & make body-writing &[u8]

Author: Mark-Simulacrum

Cargo.lock

@@ -56,10 +56,8 @@ prometheus = "0.13.3"
 cargo_metadata = "0.18.1"
 indexmap = { version = "2.0.2", features = ["serde"] }
 tokio = "1.24"
-aws-types = "0.56.1"
-aws-credential-types = "0.56.1"
-aws-smithy-async = "0.56.1"
-aws-sdk-s3 = "0.34"
+aws-sdk-s3 = "1.7"
+aws-config = { version = "1", features = ["behavior-version-latest"] }
 thiserror = "1.0.38"
 
 [dev-dependencies]

Cargo.toml

@@ -111,7 +111,7 @@ fn write_all_archive<DB: ReadResults, W: ReportWriter>(
         let len = data.len();
         match dest.write_bytes(
             "logs-archives/all.tar.gz",
-            data,
+            &data,
             &"application/gzip".parse().unwrap(),
             EncodingType::Plain,
         ) {
@@ -164,7 +164,7 @@ pub fn write_logs_archives<DB: ReadResults, W: ReportWriter>(
         let data = archive.into_inner()?.finish()?;
         dest.write_bytes(
             format!("logs-archives/{comparison}.tar.gz"),
-            data,
+            &data,
             &"application/gzip".parse().unwrap(),
             EncodingType::Plain,
         )?;

src/report/archives.rs

@@ -298,13 +298,13 @@ pub fn write_html_report<W: ReportWriter>(
     info!("copying static assets");
     dest.write_bytes(
         "report.js",
-        js_in.content()?.into_owned(),
+        &js_in.content()?,
         js_in.mime(),
         EncodingType::Plain,
     )?;
     dest.write_bytes(
         "report.css",
-        css_in.content()?.into_owned(),
+        &css_in.content()?,
         css_in.mime(),
         EncodingType::Plain,
     )?;

src/report/html.rs

@@ -293,7 +293,7 @@ fn write_logs<DB: ReadResults, W: ReportWriter>(
             s.spawn(move || {
                 while let Ok((log_path, data, encoding)) = rx.recv() {
                     if let Err(e) =
-                        dest.write_bytes(log_path, data, &mime::TEXT_PLAIN_UTF_8, encoding)
+                        dest.write_bytes(log_path, &data, &mime::TEXT_PLAIN_UTF_8, encoding)
                     {
                         errors.lock().unwrap().push(e);
                     }
@@ -548,7 +548,7 @@ pub trait ReportWriter: Send + Sync {
     fn write_bytes<P: AsRef<Path>>(
         &self,
         path: P,
-        b: Vec<u8>,
+        b: &[u8],
         mime: &Mime,
         encoding_type: EncodingType,
     ) -> Fallible<()>;
@@ -574,7 +574,7 @@ impl ReportWriter for FileWriter {
     fn write_bytes<P: AsRef<Path>>(
         &self,
         path: P,
-        b: Vec<u8>,
+        b: &[u8],
         _: &Mime,
         _: EncodingType,
     ) -> Fallible<()> {
@@ -619,7 +619,7 @@ impl ReportWriter for DummyWriter {
     fn write_bytes<P: AsRef<Path>>(
         &self,
         path: P,
-        b: Vec<u8>,
+        b: &[u8],
         mime: &Mime,
         _: EncodingType,
     ) -> Fallible<()> {

src/report/mod.rs

@@ -77,12 +77,12 @@ impl ReportWriter for S3Writer {
     fn write_bytes<P: AsRef<Path>>(
         &self,
         path: P,
-        s: Vec<u8>,
+        body: &[u8],
         mime: &Mime,
         encoding_type: EncodingType,
     ) -> Fallible<()> {
         // At least 50 MB, then use a multipart upload...
-        if s.len() >= 50 * 1024 * 1024 {
+        if body.len() >= 50 * 1024 * 1024 {
             let mut request = self
                 .client
                 .create_multipart_upload()
@@ -108,12 +108,12 @@ impl ReportWriter for S3Writer {
             };
 
             let chunk_size = 20 * 1024 * 1024;
-            let bytes = bytes::Bytes::from(s);
             let mut part = 1;
             let mut start = 0;
             let mut parts = aws_sdk_s3::types::CompletedMultipartUpload::builder();
-            while start < bytes.len() {
-                let chunk = bytes.slice(start..std::cmp::min(start + chunk_size, bytes.len()));
+            while start < body.len() {
+                let chunk = &body[start..std::cmp::min(start + chunk_size, body.len())];
+                let chunk = bytes::Bytes::copy_from_slice(chunk);
 
                 let request = self
                     .client
@@ -160,7 +160,9 @@ impl ReportWriter for S3Writer {
             let mut request = self
                 .client
                 .put_object()
-                .body(aws_sdk_s3::primitives::ByteStream::from(s))
+                .body(aws_sdk_s3::primitives::ByteStream::from(
+                    bytes::Bytes::copy_from_slice(body),
+                ))
                 .acl(aws_sdk_s3::types::ObjectCannedAcl::PublicRead)
                 .key(format!(
                     "{}/{}",
@@ -185,7 +187,7 @@ impl ReportWriter for S3Writer {
     }
 
     fn write_string<P: AsRef<Path>>(&self, path: P, s: Cow<str>, mime: &Mime) -> Fallible<()> {
-        self.write_bytes(path, s.into_owned().into_bytes(), mime, EncodingType::Plain)
+        self.write_bytes(path, s.as_bytes(), mime, EncodingType::Plain)
     }
 }
 

src/report/s3.rs

@@ -1,5 +1,3 @@
-use aws_sdk_s3::config::retry::RetryConfig;
-
 use crate::experiments::{Experiment, Status};
 use crate::prelude::*;
 use crate::report::{self, Comparison, TestResults};
@@ -17,24 +15,21 @@ use super::tokens::BucketRegion;
 const AUTOMATIC_THREAD_WAKEUP: u64 = 600;
 
 fn generate_report(data: &Data, ex: &Experiment, results: &DatabaseDB) -> Fallible<TestResults> {
-    let mut config = aws_types::SdkConfig::builder();
+    let mut config = aws_config::from_env();
     match &data.tokens.reports_bucket.region {
         BucketRegion::S3 { region } => {
-            config.set_region(Some(aws_types::region::Region::new(region.to_owned())));
+            config = config.region(aws_sdk_s3::config::Region::new(region.to_owned()));
         }
         BucketRegion::Custom { url } => {
-            config.set_region(Some(aws_types::region::Region::from_static("us-east-1")));
-            config.set_endpoint_url(Some(url.clone()));
+            config = config.region(aws_sdk_s3::config::Region::from_static("us-east-1"));
+            config = config.endpoint_url(url.clone());
         }
     }
-    config.set_credentials_provider(Some(data.tokens.reports_bucket.to_aws_credentials()));
-    // https://github.com/awslabs/aws-sdk-rust/issues/586 -- without this, the
-    // SDK will just completely not retry requests.
-    config.set_sleep_impl(Some(aws_sdk_s3::config::SharedAsyncSleep::new(
-        aws_smithy_async::rt::sleep::TokioSleep::new(),
-    )));
-    config.set_retry_config(Some(RetryConfig::standard()));
-    let config = config.build();
+    config = config.credentials_provider(data.tokens.reports_bucket.to_aws_credentials());
+    let config = tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()?
+        .block_on(config.load());
     let client = aws_sdk_s3::Client::new(&config);
     let writer = report::S3Writer::create(
         client,

src/server/reports.rs

@@ -30,17 +30,13 @@ pub struct ReportsBucket {
 }
 
 impl ReportsBucket {
-    pub(crate) fn to_aws_credentials(
-        &self,
-    ) -> aws_credential_types::provider::SharedCredentialsProvider {
-        aws_credential_types::provider::SharedCredentialsProvider::new(
-            aws_sdk_s3::config::Credentials::new(
-                self.access_key.clone(),
-                self.secret_key.clone(),
-                None,
-                None,
-                "crater-credentials",
-            ),
+    pub(crate) fn to_aws_credentials(&self) -> aws_sdk_s3::config::Credentials {
+        aws_sdk_s3::config::Credentials::new(
+            self.access_key.clone(),
+            self.secret_key.clone(),
+            None,
+            None,
+            "crater-credentials",
         )
     }
 }