Add support for copying to an output bucket directly

Author: rtyler

Cargo.toml

@@ -1,26 +1,26 @@
 [package]
 name = "lambda-s3-restructure"
-version = "0.4.0"
+version = "0.5.0"
 edition = "2021"
 homepage = "https://github.com/buoyant-data/lambda-s3-restructure"
 authors = [
     "R Tyler Croy <rtyler@buoyantdata.com>",
 ]
 
 [dependencies]
-anyhow = "1.0.75"
+anyhow = "=1"
 aws-config = { version = "1.5", features = ["behavior-version-latest"]}
-aws-sdk-s3 = "1.49"
-aws_lambda_events = { version = "0.15.1", default-features = false, features = ["sns", "sqs", "s3"] }
-chrono = "0.4.31"
+aws-sdk-s3 = "=1"
+aws_lambda_events = { version = "0.16", default-features = false, features = ["sns", "sqs", "s3"] }
+chrono = "0.4"
 liquid = "0.26"
 serde = { version = "=1", features = ["rc"] }
-lambda_runtime = "0.13.0"
-regex = "1.10"
-routefinder = "0.5.3"
-serde_json = "1.0.105"
+lambda_runtime = "0.13"
+regex = "=1"
+routefinder = "0.5"
+serde_json = "=1"
 tokio = { version = "1.40", features = ["macros"] }
 tracing = { version = "0.1", features = ["log"] }
 tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "env-filter", "tracing-log"] }
-urlencoding = "2.1.3"
+urlencoding = "2.1"
 url = { version = "2.3", features = ["serde"] }

README.adoc

@@ -66,6 +66,10 @@ The following environment variables must be set for the function to run properly
 | _required byt empty by default_
 | A link:https://crates.io/crates/liquid[liquid] template which produces a compatible path for outputting the file into the S3 bucket.
 
+| `OUTPUT_BUCKET`
+| _optional_
+| A _different_ bucket than the source bucket to execute the copy object into
+
 | `EXCLUDE_REGEX`
 | 
 | optional regular expression for keys to exclude from consideration

src/main.rs

@@ -63,7 +63,8 @@ async fn function_handler(
     let input_pattern =
         std::env::var("INPUT_PATTERN").expect("You must define INPUT_PATTERN in the environment");
     let exclude_regex: Option<Regex> = std::env::var("EXCLUDE_REGEX")
-                            .map(|ex| Regex::new(ex.as_ref()).expect("Failed to compile EXCLUDE_REGEX")).ok();
+        .map(|ex| Regex::new(ex.as_ref()).expect("Failed to compile EXCLUDE_REGEX"))
+        .ok();
     let output_template = std::env::var("OUTPUT_TEMPLATE")
         .expect("You must define OUTPUT_TEMPLATE in the environment");
 
@@ -91,10 +92,11 @@ async fn function_handler(
             let output_key = template.render(&parameters)?;
             info!("Copying {source_key:?} to {output_key:?}");
             if let Some(bucket) = entity.bucket.name {
-                debug!("Sending a copy request for {bucket} with {source_key} to {output_key}");
+                let output_bucket = std::env::var("OUTPUT_BUCKET").unwrap_or(bucket.clone());
+                debug!("Sending a copy request for {output_bucket} with {bucket}/{source_key} to {output_key}");
                 let result = client
                     .copy_object()
-                    .bucket(&bucket)
+                    .bucket(&output_bucket)
                     .copy_source(format!("{bucket}/{source_key}"))
                     .key(output_key)
                     .send()
@@ -291,16 +293,21 @@ mod tests {
 
     #[test]
     fn test_exclude_regex() {
-        let exclude = Some(Regex::new(r#"^path\/to\/table.*"#).expect("Failed to compile regular expression"));
+        let exclude = Some(
+            Regex::new(r#"^path\/to\/table.*"#).expect("Failed to compile regular expression"),
+        );
         let keys = vec![
             "path/to/alpha",
             "path/to/bravo/foo.parquet",
             "path/to/table",
             "path/to/table/foo.parquet",
         ];
 
-        let filtered: Vec<_> = keys.iter().filter(|k| !should_exclude(exclude.as_ref(), k)).map(|k| k.clone()).collect();
+        let filtered: Vec<_> = keys
+            .iter()
+            .filter(|k| !should_exclude(exclude.as_ref(), k))
+            .map(|k| k.clone())
+            .collect();
         assert_ne!(filtered, keys);
     }
 }
-