Implement deployment using S3 (Create a bucket for each region.) (#455)

* Add class for uploading deploy package to S3

* Sort regions

* Fix to set region at client creation

Currently, a region is set to the aws instance to be used in common.
Since it may not be as expected as it is in parallel processing, it is
set when each client is created.

* Implement deployment function using S3

Also create buckets

* Update README

Author: abetomo

README.md

@@ -93,8 +93,7 @@ SRC_DIRECTORY              // (default: '')
 DEPLOY_TIMEOUT             // (default: '120000')
 DOCKER_IMAGE               // (default: '')
 DEPLOY_ZIPFILE             // (default: '')
-DEPLOY_S3_BUCKET           // (default: '')
-DEPLOY_S3_KEY              // (default: '')
+DEPLOY_USE_S3              // (default: false)
 AWS_DLQ_TARGET_ARN         // (default: not set)
 ```
 
@@ -190,8 +189,7 @@ Options:
   -D, --prebuiltDirectory []          Prebuilt directory
   -T, --deployTimeout [120000]        Deploy Timeout
   -z, --deployZipfile []              Deploy zipfile
-  -B, --deployS3Bucket []             Use S3 to deploy. Specify it with `--deployS3Key`. (default: )
-  -O, --deployS3Key []                Use S3 to deploy. Specify it with `--deployS3Bucket`. (default: )
+  -B, --deployUseS3 []                Use S3 to deploy.
   -y, --proxy []                      Proxy server
 ```
 

bin/node-lambda

@@ -46,8 +46,7 @@ const SRC_DIRECTORY = process.env.SRC_DIRECTORY || ''
 const DEPLOY_TIMEOUT = process.env.DEPLOY_TIMEOUT || 120000
 const DOCKER_IMAGE = process.env.DOCKER_IMAGE || ''
 const DEPLOY_ZIPFILE = process.env.DEPLOY_ZIPFILE || ''
-const DEPLOY_S3_BUCKET = process.env.DEPLOY_S3_BUCKET || ''
-const DEPLOY_S3_KEY = process.env.DEPLOY_S3_KEY || ''
+const DEPLOY_USE_S3 = process.env.DEPLOY_USE_S3 || false
 const AWS_KMS_KEY_ARN = process.env.AWS_KMS_KEY_ARN || ''
 const AWS_DLQ_TARGET_ARN = (() => {
   // You can clear the setting by passing an empty string
@@ -103,10 +102,7 @@ program
   .option('-D, --prebuiltDirectory [PREBUILT_DIRECTORY]', 'Prebuilt directory', PREBUILT_DIRECTORY)
   .option('-T, --deployTimeout [DEPLOY_TIMEOUT]', 'Deploy Timeout', DEPLOY_TIMEOUT)
   .option('-z, --deployZipfile [DEPLOY_ZIPFILE]', 'Deploy zipfile', DEPLOY_ZIPFILE)
-  .option('-B, --deployS3Bucket [DEPLOY_S3_BUCKET]',
-    'Use S3 to deploy. Specify it with `--deployS3Key`.', DEPLOY_S3_BUCKET)
-  .option('-O, --deployS3Key [DEPLOY_S3_KEY]',
-    'Use S3 to deploy. Specify it with `--deployS3Bucket`.', DEPLOY_S3_KEY)
+  .option('-B, --deployUseS3 [DEPLOY_USE_S3]', 'Use S3 to deploy.', DEPLOY_USE_S3)
   .option('-y, --proxy [PROXY]', 'Proxy server', PROXY)
   .action((prg) => lambda.deploy(prg))
 

lib/cloudwatch_logs.js

@@ -1,9 +1,10 @@
 'use strict'
 
 class CloudWatchLogs {
-  constructor (aws) {
+  constructor (aws, region) {
     // Authenticated `aws` object in `lib/main.js`
     this.lambda = new aws.Lambda({
+      region: region,
       apiVersion: '2015-03-31'
     })
     this.cloudwatchlogs = new aws.CloudWatchLogs({

lib/main.js

@@ -13,6 +13,7 @@ const dotenv = require('dotenv')
 const proxy = require('proxy-agent')
 const ScheduleEvents = require(path.join(__dirname, 'schedule_events'))
 const S3Events = require(path.join(__dirname, 's3_events'))
+const S3Deploy = require(path.join(__dirname, 's3_deploy'))
 const CloudWatchLogs = require(path.join(__dirname, 'cloudwatch_logs'))
 
 const AWSXRay = require('aws-xray-sdk-core')
@@ -177,10 +178,10 @@ so you can easily test run multiple events.
   }
 
   _isUseS3 (program) {
-    return program.deployS3Bucket != null &&
-      program.deployS3Key != null &&
-      program.deployS3Bucket.length > 0 &&
-      program.deployS3Key.length > 0
+    if (typeof program.deployUseS3 === 'boolean') {
+      return program.deployUseS3
+    }
+    return program.deployUseS3 === 'true'
   }
 
   _params (program, buffer) {
@@ -219,8 +220,8 @@ so you can easily test run multiple events.
 
     if (this._isUseS3(program)) {
       params.Code = {
-        S3Bucket: program.deployS3Bucket,
-        S3Key: program.deployS3Key
+        S3Bucket: null,
+        S3Key: null
       }
     } else {
       params.Code = { ZipFile: buffer }
@@ -779,23 +780,6 @@ they may not work as expected in the Lambda environment.
     })
   }
 
-  _s3PutObject (program, region, buffer) {
-    this._awsConfigUpdate(program, region)
-    const s3 = new aws.S3()
-    return new Promise((resolve, reject) => {
-      const params = {
-        Body: buffer,
-        Bucket: program.deployS3Bucket,
-        Key: program.deployS3Key
-      }
-      console.log(`=> Uploading zip file to S3 ${region}`)
-      s3.putObject(params, (err, data) => {
-        if (err) reject(err)
-        resolve(data)
-      })
-    })
-  }
-
   _awsConfigUpdate (program, region) {
     const awsSecurity = { region: region }
 
@@ -832,23 +816,35 @@ they may not work as expected in the Lambda environment.
       !!err.message.match(/^Function not found:/)
   }
 
-  _deployToRegion (program, params, region) {
+  _deployToRegion (program, params, region, buffer) {
+    this._awsConfigUpdate(program, region)
+
     console.log('=> Reading event source file to memory')
     const eventSourceList = this._eventSourceList(program)
 
     return Promise.resolve().then(() => {
+      if (this._isUseS3(program)) {
+        const s3Deploy = new S3Deploy(aws, region)
+        return s3Deploy.putPackage(params, region, buffer)
+      }
+      return null
+    }).then((code) => {
+      if (code != null) params.Code = code
+    }).then(() => {
       if (!this._isUseS3(program)) {
         console.log(`=> Uploading zip file to AWS Lambda ${region} with parameters:`)
       } else {
         console.log(`=> Uploading AWS Lambda ${region} with parameters:`)
       }
       console.log(params)
 
-      this._awsConfigUpdate(program, region)
-      const lambda = new aws.Lambda({ apiVersion: '2015-03-31' })
-      const scheduleEvents = new ScheduleEvents(aws)
-      const s3Events = new S3Events(aws)
-      const cloudWatchLogs = new CloudWatchLogs(aws)
+      const lambda = new aws.Lambda({
+        region: region,
+        apiVersion: '2015-03-31'
+      })
+      const scheduleEvents = new ScheduleEvents(aws, region)
+      const s3Events = new S3Events(aws, region)
+      const cloudWatchLogs = new CloudWatchLogs(aws, region)
 
       // Checking function
       return lambda.getFunction({
@@ -945,15 +941,17 @@ they may not work as expected in the Lambda environment.
     const regions = program.region.split(',')
     return this._archive(program).then((buffer) => {
       console.log('=> Reading zip file to memory')
-      if (this._isUseS3(program)) {
-        return this._s3PutObject(program, regions[0], buffer)
-      }
       return buffer
     }).then((buffer) => {
       const params = this._params(program, buffer)
 
       return Promise.all(regions.map((region) => {
-        return this._deployToRegion(program, params, region)
+        return this._deployToRegion(
+          program,
+          params,
+          region,
+          this._isUseS3(program) ? buffer : null
+        )
       })).then(results => {
         this._printDeployResults(results, true)
       })

lib/s3_events.js

@@ -5,12 +5,14 @@
  * Put the Notification Configuration in the existing Bucket.
  */
 class S3Events {
-  constructor (aws) {
+  constructor (aws, region) {
     // Authenticated `aws` object in `lib/main.js`
     this.lambda = new aws.Lambda({
+      region: region,
       apiVersion: '2015-03-31'
     })
     this.s3 = new aws.S3({
+      region: region,
       apiVersion: '2006-03-01'
     })
   }

lib/schedule_events.js

@@ -1,9 +1,10 @@
 'use strict'
 
 class ScheduleEvents {
-  constructor (aws) {
+  constructor (aws, region) {
     // Authenticated `aws` object in `lib/main.js`
     this.lambda = new aws.Lambda({
+      region: region,
       apiVersion: '2015-03-31'
     })
     this.cloudwatchevents = new aws.CloudWatchEvents({

test/main.js

@@ -194,18 +194,16 @@ describe('lib/main', function () {
 
   describe('_isUseS3', () => {
     it('=== true', () => {
-      assert.isTrue(lambda._isUseS3({
-        deployS3Bucket: 'bucket',
-        deployS3Key: 'key'
-      }))
+      assert.isTrue(lambda._isUseS3({deployUseS3: true}))
+      assert.isTrue(lambda._isUseS3({deployUseS3: 'true'}))
     })
 
     it('=== false', () => {
       [
         {},
-        {deployS3Bucket: '', deployS3Key: ''},
-        {deployS3Bucket: 'bucket'},
-        {deployS3Key: 'key'}
+        {deployUseS3: false},
+        {deployUseS3: 'false'},
+        {deployUseS3: 'foo'}
       ].forEach((params) => {
         assert.isFalse(lambda._isUseS3(params), params)
       })
@@ -310,15 +308,12 @@ describe('lib/main', function () {
       })
 
       it('Use S3 deploy', () => {
-        const params = lambda._params(Object.assign({
-          deployS3Bucket: 'S3Bucket-value',
-          deployS3Key: 'S3Key-value'
-        }, program), 'Buffer')
+        const params = lambda._params(Object.assign({deployUseS3: true}, program), 'Buffer')
         assert.deepEqual(
           params.Code,
           {
-            S3Bucket: 'S3Bucket-value',
-            S3Key: 'S3Key-value'
+            S3Bucket: null,
+            S3Key: null
           }
         )
       })
@@ -1246,20 +1241,6 @@ describe('lib/main', function () {
     })
   })
 
-  describe('Lambda.prototype._s3PutObject()', () => {
-    it('simple test with mock', () => {
-      disableLog()
-      const params = {
-        deployS3Bucket: 'test',
-        deployS3Key: 'test'
-      }
-      return lambda._s3PutObject(params, 'us-east-1', 'buffer').then((result) => {
-        enableLog()
-        assert.deepEqual(result, {'test': 'putObject'})
-      })
-    })
-  })
-
   describe('Lambda.prototype._deployToRegion()', () => {
     it('simple test with mock', () => {
       const params = lambda._params(program, null)