Update to version v2.3.0

Author: YikaiHu

CHANGELOG.md

@@ -4,7 +4,92 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2.3.0] - 2023-03-30
+- Support S3 Access Key Rotation
 
-## [1.0.0] - 2022-01-04
+## [2.0.2] - 2021-06-19
+
+### Changed
+
+- Use Secrets Manager to store credentials
+- Use container image on Amazon ECR Public Gallery
+
+### Added
+- Implement auto shutdown on EC2 instances when error occurred
+
+### Fixed
+
+- Potential out of memory issue on EC2 when transferring large files
+- Incorrect part number and chunk size for file over 50GB
+
+
+
+## [2.0.1] - 2021-04-23
+
+### Changed
+
+- Support custom endpoint url
+- Support object ACL
+
+
+## [2.0.0] - 2021-03-17
+
+### Changed
+
+- Use EC2 + Auto Scaling Group to replace Lambda to do data transfer
+- Rewrite core logic in golang (Separate project)
+- Support cross account deployment
+
+
+## [1.3.0] - 2020-12-30
+
+### Changed
+- Regroup the cloudformation parameters
+
+### Added
+- Add support of S3 Delete Event
+
+## [1.2.0] - 2020-12-24
+
+### Fixed
+- Fix wrong metric name in Lambda-NETWORK widget
+
+### Changed
+- Use S3 Native SDK to access and get objects from Aliyun OSS
+- Region name is now one of the stack parameters. Aligned with ECR plugin.
+
+### Added
+- Add support of replicating from Google Cloud Storage to Amazon S3 (Global)
+
+## [1.1.0] - 2020-12-21
+
+### Changed
+- Use custom provider to handling stack events
+
+### Added
+- Add support of triggering replication base on S3 Event.
+
+## [1.0.2] - 2020-12-06
+
+### Changed
+- Change to use CDK v1.74.0
+- Reduce the number of logs generated.
+
+### Added
+- Add support of accessing s3 with no-sign-request
+
+## [1.0.1] - 2020-11-16
+### Added
+- Advanced options to control replication process, such as lambda memory, chunk size etc.
+- Support of choosing different destination storage class
+
+### Fixed
+- Cloudformation parameters are not grouped and ordered
+
+### Changed
+- ECR image is now tagged with version number.
+
+
+## [1.0.0] - 2020-09-30
 ### Added
 - All files, initial version

CONTRIBUTING.md

@@ -56,4 +56,4 @@ If you discover a potential security issue in this project we ask that you notif
 
 ## Licensing
 
-See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution.
+See the [LICENSE](https://github.com/awslabs/amazon-s3-data-replication-hub-plugin/blob/main/LICENSE.txt) file for our project's licensing. We will ask you to confirm the licensing of your contribution.

NOTICE.txt

@@ -1,14 +1,21 @@
 Data Transfer Hub - S3 Plugin
-Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except
-in compliance with the License. A copy of the License is located at http://www.apache.org/licenses/
-or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. See the License for the
-specific language governing permissions and limitations under the License.
+Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 
 **********************
 THIRD PARTY COMPONENTS
 **********************
 This software includes third party software subject to the following copyrights:
 
-AWS SDK under the Apache License Version 2.0
+aws-cdk under the Apache License 2.0
+aws-cdk-lib under the Apache License 2.0
+cdk-nag under the Apache License 2.0
+constructs under the Apache License 2.0
+source-map-support under the Massachusetts Institute of Technology (MIT) license
+@types/jest under the Massachusetts Institute of Technology (MIT) license
+@types/node under the Massachusetts Institute of Technology (MIT) license
+aws-sdk under the Apache License 2.0
+jest under the Massachusetts Institute of Technology (MIT) license
+path-parse under the Massachusetts Institute of Technology (MIT) license
+ts-jest under the Massachusetts Institute of Technology (MIT) license
+ts-node under the Massachusetts Institute of Technology (MIT) license
+typescript under the Apache License 2.0

deployment/build-s3-dist.sh

@@ -47,10 +47,10 @@ if [ -z "$1" ] || [ -z "$2" ] ; then
 fi
 
 # Get default version
-if [ -z "$3" ]; then
-    export VERSION=$(git describe --tags || echo v0.0.0)
+if [ ! -z $3 ]; then
+    export VERSION="$3"
 else
-    export VERSION=$3
+    export VERSION=$(git describe --tags --exact-match || { [ -n "$BRANCH_NAME" ] && echo "$BRANCH_NAME"; } || echo v0.0.0)
 fi
 
 # Get reference for all important folders
@@ -67,7 +67,7 @@ echo "--------------------------------------------------------------------------
 echo AWS_DEFAULT_REGION $AWS_DEFAULT_REGION
 echo DIST_OUTPUT_BUCKET $1
 echo SOLUTION_NAME $2
-echo REGION $REGION
+# echo REGION $REGION
 
 echo "------------------------------------------------------------------------------"
 echo "[Init] Remove any old dist files from previous runs"
@@ -103,10 +103,6 @@ echo "--------------------------------------------------------------------------
 echo "cd $source_dir"
 cd $source_dir
 
-# Install and build
-echo "npm run build"
-npm run build
-
 # Run 'cdk synth' to generate raw solution outputs
 echo "cdk synth --output=$staging_dist_dir"
 npx cdk synth -j --output=$staging_dist_dir -c runType=EC2 > $staging_dist_dir/DataTransferS3Stack-ec2.template.json
@@ -181,22 +177,39 @@ for d in `find . -mindepth 1 -maxdepth 1 -type d`; do
     # cd $d
     # Rename the artifact, removing the period for handler compatibility
     pfname="$(basename -- $d)"
-    fname="$(echo $pfname | sed -e 's/\.//g')"
-    echo "zipping the artifact"
-    mv $d $fname
-    pushd $fname
-    echo "zip -qr9 $staging_dist_dir/$fname.zip ."
-    zip -qr9 $staging_dist_dir/$fname.zip .
-    popd
-
-    # Copy the zipped artifact from /staging to /regional-s3-assets
-    echo "mv $fname.zip $build_dist_dir"
-    mv $fname.zip $build_dist_dir
+    # zip folder
+    echo "zip -rq $pfname.zip $pfname"
+    cd $pfname
+    zip -rq $pfname.zip *
+    mv $pfname.zip ../
+    cd ..
 
     # Remove the old, unzipped artifact from /staging
-    echo "rm -rf $fname"
-    rm -rf $fname
+    echo "rm -rf $pfname"
+    rm -rf $pfname
+
+    # ... repeat until all source code artifacts are zipped and placed in the /staging
+done
+
+
+# ... For each asset.*.zip code artifact in the temporary /staging folder...
+cd $staging_dist_dir
+for f in `find . -iname \*.zip`; do
+    # Rename the artifact, removing the period for handler compatibility
+    # pfname = asset.<key-name>.zip
+    pfname="$(basename -- $f)"
+    echo $pfname
+    # fname = <key-name>.zip
+    fname="$(echo $pfname | sed -e 's/asset\.//g')"
+    mv $pfname $fname
+
+    # Copy the zipped artifact from /staging to /regional-s3-assets
+    echo "cp $fname $build_dist_dir"
+    cp $fname $build_dist_dir
 
+    # Remove the old, zipped artifact from /staging
+    echo "rm $fname"
+    rm $fname
 done
 
 echo "------------------------------------------------------------------------------"

deployment/cdk-solution-helper/index.js

@@ -33,12 +33,13 @@ fs.readdirSync(global_s3_assets).forEach(file => {
     const fn = template.Resources[f];
     if (fn.Properties.Code.hasOwnProperty('S3Bucket')) {
       // Set the S3 key reference
-      let artifactHash = Object.assign(fn.Properties.Code.S3Bucket.Ref);
-      // artifactHash = artifactHash.replace('AssetParameters', '');
-      let start = artifactHash.indexOf('AssetParameters') + 15
-      artifactHash = artifactHash.substring(start, artifactHash.indexOf('S3Bucket'));
-      const assetPath = `asset${artifactHash}`;
-      fn.Properties.Code.S3Key = `%%SOLUTION_NAME%%/%%VERSION%%/${assetPath}.zip`;
+      let s3Key = Object.assign(fn.Properties.Code.S3Key);
+      // https://github.com/aws/aws-cdk/issues/10608
+      if (!s3Key.endsWith('.zip')) {
+        fn.Properties.Code.S3Key = `%%SOLUTION_NAME%%/%%VERSION%%/${s3Key}.zip`;
+      } else {
+        fn.Properties.Code.S3Key = `%%SOLUTION_NAME%%/%%VERSION%%/${s3Key}`;
+      }
       // Set the S3 bucket reference
       fn.Properties.Code.S3Bucket = {
         'Fn::Sub': '%%BUCKET_NAME%%-${AWS::Region}'
@@ -67,58 +68,20 @@ fs.readdirSync(global_s3_assets).forEach(file => {
     }
   });
 
-  // Clean-up Lambda layer code dependencies
+  // Clean-up Lambda Layer code dependencies
   const lambdaLayers = Object.keys(resources).filter(function (key) {
     return resources[key].Type === "AWS::Lambda::LayerVersion";
-  });
-  lambdaLayers.forEach(function (f) {
-    const fn = template.Resources[f];
-    if (fn.Properties.Content.hasOwnProperty('S3Bucket')) {
-      // Set the S3 key reference
-      let artifactHash = Object.assign(fn.Properties.Content.S3Bucket.Ref);
-      // artifactHash = artifactHash.replace('AssetParameters', '');
-      let start = artifactHash.indexOf('AssetParameters') + 15
-      artifactHash = artifactHash.substring(start, artifactHash.indexOf('S3Bucket'));
-      const assetPath = `asset${artifactHash}`;
-      fn.Properties.Content.S3Key = `%%SOLUTION_NAME%%/%%VERSION%%/${assetPath}.zip`;
-      // Set the S3 bucket reference
-      fn.Properties.Content.S3Bucket = {
+  })
+  lambdaLayers.forEach(function (l) {
+    const layer = template.Resources[l];
+    if (layer.Properties.Content.hasOwnProperty('S3Bucket')) {
+      let s3Key = Object.assign(layer.Properties.Content.S3Key);
+      layer.Properties.Content.S3Key = `%%SOLUTION_NAME%%/%%VERSION%%/${s3Key}`;
+      layer.Properties.Content.S3Bucket = {
         'Fn::Sub': '%%BUCKET_NAME%%-${AWS::Region}'
-      };
-      // // Set the handler
-      // const handler = fn.Properties.Handler;
-      // fn.Properties.Handler = `${assetPath}/${handler}`;
-    }
-  });
-
-  // Clean-up nested template stack dependencies
-  const nestedStacks = Object.keys(resources).filter(function (key) {
-    return resources[key].Type === 'AWS::CloudFormation::Stack'
-  });
-  nestedStacks.forEach(function (f) {
-    const fn = template.Resources[f];
-    fn.Properties.TemplateURL = {
-      'Fn::Join': [
-        '',
-        [
-          fn.Metadata.domain,
-          '/',
-          `%%SOLUTION_NAME%%/%%VERSION%%/${fn.Metadata.nestedTemplateName}`
-        ]
-      ]
-    };
-
-    const params = fn.Properties.Parameters ? fn.Properties.Parameters : {};
-    const nestedStackParameters = Object.keys(params).filter(function (key) {
-      if (key.search(/[\w]*AssetParameters/g) > -1) {
-        return true;
       }
-      return false;
-    });
-    nestedStackParameters.forEach(function (stkParam) {
-      fn.Properties.Parameters[stkParam] = undefined;
-    });
-  });
+    }
+  })
 
 
   // Clean-up parameters section
@@ -130,6 +93,17 @@ fs.readdirSync(global_s3_assets).forEach(file => {
     template.Parameters[a] = undefined;
   });
 
+  // Clean-up BootstrapVersion parameter
+  if (parameters.hasOwnProperty('BootstrapVersion')) {
+    parameters.BootstrapVersion = undefined
+  }
+
+  // Clean-up CheckBootstrapVersion Rule
+  const rules = (template.Rules) ? template.Rules : {};
+  if (rules.hasOwnProperty('CheckBootstrapVersion')) {
+    rules.CheckBootstrapVersion = undefined
+  }
+
   // Output modified template file
   const output_template = JSON.stringify(template, null, 2);
   fs.writeFileSync(`${global_s3_assets}/${file}`, output_template);

docs/DEPLOYMENT_CN.md

@@ -64,6 +64,7 @@
       [![Launch Stack](launch-stack.svg)](https://console.aws.amazon.com/cloudformation/home#/stacks/create/template?stackName=DTHS3Stack&templateURL=https://solutions-reference.s3.amazonaws.com/data-transfer-hub-s3-plugin/latest/DataTransferS3Stack-ec2.template)
 
 
+
 1. 单击**下一步**。 相应地为参数指定值。 如果需要，请更改堆栈名称。
 
 1. 单击**下一步**。 配置其他堆栈选项，例如标签（可选）。

source/bin/main-stack.ts

@@ -18,8 +18,31 @@ limitations under the License.
 
 
 import 'source-map-support/register';
-import * as cdk from '@aws-cdk/core';
+import { App, Aspects, Stack } from "aws-cdk-lib";
 import { DataTransferS3Stack } from '../lib/main-stack';
 
-const app = new cdk.App();
-new DataTransferS3Stack(app, 'DataTransferS3Stack');
+import {
+    AwsSolutionsChecks,
+    NagPackSuppression,
+    NagSuppressions,
+} from "cdk-nag";
+
+const app = new App();
+
+function stackSuppressions(
+    stacks: Stack[],
+    suppressions: NagPackSuppression[]
+) {
+    stacks.forEach((s) =>
+        NagSuppressions.addStackSuppressions(s, suppressions, true)
+    );
+}
+
+stackSuppressions([
+    new DataTransferS3Stack(app, 'DataTransferS3Stack'),
+], [
+    { id: 'AwsSolutions-IAM5', reason: 'some policies need to get dynamic resources' },
+    { id: 'AwsSolutions-IAM4', reason: 'these policies is used by CDK Customer Resource lambda' },
+]);
+
+Aspects.of(app).add(new AwsSolutionsChecks());

source/cdk.json

@@ -1,7 +1,6 @@
 {
   "app": "npx ts-node bin/main-stack.ts",
   "context": {
-    "@aws-cdk/core:enableStackNameDuplicates": "true",
     "aws-cdk:enableDiffNoFail": "true"
   }
 }

source/lambda/.coveragerc

@@ -0,0 +1,9 @@
+[run]
+omit =
+    tests/*
+    .venv-*/*
+    test/*
+    */__init__.py
+    assets/*
+source =
+    .