Added --prebuiltDirectory (#116)

* added --prebuiltDirectory

* Use rsync to copy necessary files before archiving

* Read package.json's name only if it exists

When using webpack, we may have one npm package and multiple lambda
functions sharing code. Lambda functions don't necessarily have a
package.json in them.

* Bug fix. All tests pass now.

Author: Sean Shirazi

README.md

@@ -68,7 +68,7 @@ $ node-lambda run --help
     -h, --help                          Output usage information
     --handler [index.handler]           Lambda Handler {index.handler}
     -j, --eventFile [event.json]        Event JSON File
-    -u, --runtime [nodejs4.3]           Lambda Runtime {nodejs4.3, nodejs} - "nodejs4.3" is the current standard, "nodejs" is v0.10.36 
+    -u, --runtime [nodejs4.3]           Lambda Runtime {nodejs4.3, nodejs} - "nodejs4.3" is the current standard, "nodejs" is v0.10.36
     -x, --contextFile [context.json]    Context JSON file
 ```
 
@@ -89,6 +89,8 @@ $ node-lambda package --help
     -e, --environment [staging]         Choose environment {development, staging, production}
     -f, --configFile []                 Path to file holding secret environment variables (e.g. "deploy.env")
     -x, --excludeGlobs []               Add a space separated list of file(type)s to ignore (e.g. "*.json .env")
+    -P, --prebuiltDirectory []          Prebuilt directory
+
 ```
 
 #### deploy
@@ -114,22 +116,23 @@ $ node-lambda deploy --help
     -m, --memorySize [128]            Lambda Memory Size
     -t, --timeout [3]                 Lambda Timeout
     -d, --description [missing]       Lambda Description
-    -u, --runtime [nodejs4.3]         Lambda Runtime {nodejs4.3, nodejs} - "nodejs4.3" is the current standard, "nodejs" is v0.10.36 
+    -u, --runtime [nodejs4.3]         Lambda Runtime {nodejs4.3, nodejs} - "nodejs4.3" is the current standard, "nodejs" is v0.10.36
     -p, --publish [false]             This boolean parameter can be used to request AWS Lambda to create the Lambda function and publish a version as an atomic operation
     -v, --version [custom-version]    Lambda Version
     -f, --configFile []               Path to file holding secret environment variables (e.g. "deploy.env")
     -b, --vpcSubnets []               VPC Subnet ID(s, comma separated list) for your Lambda Function, when using this, the below param is also required
     -g, --vpcSecurityGroups []        VPC Security Group ID(s, comma separated list) for your Lambda Function, when using this, the above param is also required
     -x, --excludeGlobs []             Add a space separated list of file(type)s to ignore (e.g. "*.json .env")
+    -P, --prebuiltDirectory []        Prebuilt directory
 ```
 
 ## Custom Environment Variables
 
-AWS Lambda doesn't let you set environment variables for your function, but in many cases you will need to configure your function with secure values that you don't want to check into version control, for example a DB connection string or encryption key. Use the sample `deploy.env` file in combination with the `--configFile` flag to set values which will be prepended to your compiled Lambda function as `process.env` environment variables before it gets uploaded to S3. 
+AWS Lambda doesn't let you set environment variables for your function, but in many cases you will need to configure your function with secure values that you don't want to check into version control, for example a DB connection string or encryption key. Use the sample `deploy.env` file in combination with the `--configFile` flag to set values which will be prepended to your compiled Lambda function as `process.env` environment variables before it gets uploaded to S3.
 
 ## Node.js Runtime Configuration
 
-AWS Lambda now supports Node.js v4.3.2, and there have been some [API changes](http://docs.aws.amazon.com/lambda/latest/dg/nodejs-prog-model-using-old-runtime.html) for the new version.  Most notably, 
+AWS Lambda now supports Node.js v4.3.2, and there have been some [API changes](http://docs.aws.amazon.com/lambda/latest/dg/nodejs-prog-model-using-old-runtime.html) for the new version.  Most notably,
 `context.done()`, `context.succeed()`, and `context.fail()` are deprecated in favor of the Node convention of passing in
 a callback function.  These will still work for now for backward compatibility, but are no longer recommended.
 
@@ -138,6 +141,12 @@ v0.10.36 is still supported, and can be targeted by changing the `AWS_RUNTIME` v
 ## Post install script
 When running `node-lambda deploy` if you need to do some action after `npm install --production` and before deploying to AWS Lambda (i.e. replace some modules with precompiled ones or download some libraries) you can create `post_install.sh` script. If the file exists the script will be executed (and output shown after execution) if not it is skipped. Make sure that the script is executable.
 
+## Prebuilt packages
+The `--prebuiltDirectory` flag is useful for working with Webpack for example. It skips `npm install --production` and `post_install.sh` and simply packages the specified directory.
+
+## Handling `npm link` and Dependencies With Local Paths
+Perhaps the easiest way to handle these cases is to bundle the code using Webpack and use the `--prebuiltDirectory` flag to package the output for deployment.
+
 ## Other AWS Lambda Tools Projects
 
 + [lambdaws](https://github.com/mentum/lambdaws)

bin/node-lambda

@@ -3,7 +3,9 @@
 var dotenv = require('dotenv');
 var lambda = require('../lib/main.js');
 var program = require('commander');
-var packageJson = require(process.cwd() + '/package.json');
+var fs = require('fs');
+var packageJsonName = fs.existsSync(process.cwd() + '/package.json')
+    ? require(process.cwd() + '/package.json').name : 'UnnamedFunction';
 
 dotenv.load();
 
@@ -14,7 +16,7 @@ var AWS_ACCESS_KEY_ID = process.env.AWS_ACCESS_KEY_ID;
 var AWS_SECRET_ACCESS_KEY = process.env.AWS_SECRET_ACCESS_KEY;
 var AWS_SESSION_TOKEN = process.env.AWS_SESSION_TOKEN || '';
 var AWS_REGION = process.env.AWS_REGION || 'us-east-1,us-west-2,eu-west-1';
-var AWS_FUNCTION_NAME = process.env.AWS_FUNCTION_NAME || packageJson.name;
+var AWS_FUNCTION_NAME = process.env.AWS_FUNCTION_NAME || packageJsonName;
 var AWS_HANDLER = process.env.AWS_HANDLER || 'index.handler';
 var AWS_ROLE = process.env.AWS_ROLE_ARN || process.env.AWS_ROLE || 'missing';
 var AWS_MEMORY_SIZE = process.env.AWS_MEMORY_SIZE || 128;
@@ -28,6 +30,9 @@ var AWS_VPC_SECURITY_GROUPS = process.env.AWS_VPC_SECURITY_GROUPS || '';
 var EVENT_FILE = process.env.EVENT_FILE || 'event.json';
 var PACKAGE_DIRECTORY = process.env.PACKAGE_DIRECTORY;
 var CONTEXT_FILE = process.env.CONTEXT_FILE || 'context.json';
+var PREBUILT_DIRECTORY = process.env.PREBUILT_DIRECTORY || '';
+
+
 
 program
   .version(lambda.version)
@@ -56,6 +61,7 @@ program
     'Path to file holding secret environment variables (e.g. "deploy.env")', CONFIG_FILE)
   .option('-x, --excludeGlobs [' + EXCLUDE_GLOBS + ']',
     'Space-separated glob pattern(s) for additional exclude files (e.g. "event.json dotenv.sample")', EXCLUDE_GLOBS)
+  .option('-P, --prebuiltDirectory [' + PREBUILT_DIRECTORY + ']', 'Prebuilt directory', PREBUILT_DIRECTORY)
   .action(function (prg) {
     lambda.deploy(prg);
   });
@@ -73,6 +79,7 @@ program
     'Space-separated glob pattern(s) for additional exclude files (e.g. "event.json dotenv.sample")', EXCLUDE_GLOBS)
   .option('-f, --configFile [' + CONFIG_FILE + ']',
     'Path to file holding secret environment variables (e.g. "deploy.env")', CONFIG_FILE)
+  .option('-P, --prebuiltDirectory [' + PREBUILT_DIRECTORY + ']', 'Prebuilt directory', PREBUILT_DIRECTORY)
   .action(function (prg) {
     lambda.package(prg);
   });

lib/main.js

@@ -137,7 +137,7 @@ Lambda.prototype._zipfileTmpPath = function (program) {
   return zipfile;
 };
 
-Lambda.prototype._rsync = function (program, codeDirectory, callback) {
+Lambda.prototype._rsync = function (program, src, dest, callback) {
   var excludes = ['.git*', '*.swp', '.editorconfig', 'deploy.env', '*.log', 'build/', 'node_modules'],
       excludeGlobs = [];
   if (program.excludeGlobs) {
@@ -147,12 +147,14 @@ Lambda.prototype._rsync = function (program, codeDirectory, callback) {
     return '--exclude=' + exclude;
   }).join(' ');
 
-  exec('mkdir -p ' + codeDirectory, function(err) {
+  exec('mkdir -p ' + dest, function(err) {
     if (err) {
       return callback(err);
     }
 
-    exec('rsync -r ' + excludeArgs + ' . ' + codeDirectory, function (err) {
+    // we need the extra / after src to make sure we are copying the content
+    // of the directory, not the directory itself.
+    exec('rsync -r ' + excludeArgs + ' ' + src.trim() + '/ ' + dest, function (err) {
       if (err) {
         return callback(err);
       }
@@ -326,6 +328,33 @@ Lambda.prototype._uploadNew = function(lambda, params, cb) {
 };
 
 Lambda.prototype._archive = function (program, archive_callback) {
+  return program.prebuiltDirectory
+    ? this._archivePrebuilt(program, archive_callback)
+    : this._buildAndArchive(program, archive_callback);
+}
+
+Lambda.prototype._archivePrebuilt = function (program, archive_callback) {
+
+  var codeDirectory = this._codeDirectory(program)
+  var _this = this;
+  this._rsync(program, program.prebuiltDirectory, codeDirectory, function (err) {
+    if (err) {
+      return archive_callback(err);
+    }
+
+    // Add custom environment variables if program.configFile is defined
+    if (program.configFile) {
+      _this._setEnvironmentVars(program, codeDirectory);
+    }
+    console.log('=> Zipping deployment package');
+    var archive = process.platform !== 'win32' ? _this._nativeZip : _this._zip;
+    archive = archive.bind(_this);
+
+    archive(program, codeDirectory, archive_callback);
+  });
+}
+
+Lambda.prototype._buildAndArchive = function (program, archive_callback) {
   this._createSampleFile('.env', '.env');
 
   // Warn if not building on 64-bit linux
@@ -345,7 +374,7 @@ Lambda.prototype._archive = function (program, archive_callback) {
     }
     console.log('=> Moving files to temporary directory');
     // Move files to tmp folder
-    _this._rsync(program, codeDirectory, function (err) {
+    _this._rsync(program, '.', codeDirectory, function (err) {
       if (err) {
         return archive_callback(err);
       }
@@ -401,6 +430,7 @@ Lambda.prototype.package = function (program) {
     if (err) {
       throw err;
     }
+
     var basename = program.functionName + (program.environment ? '-' + program.environment : '');
     var zipfile = path.join(program.packageDirectory, basename + '.zip');
     console.log('=> Writing packaged zip');
@@ -464,4 +494,5 @@ Lambda.prototype.deploy = function (program) {
   });
 };
 
+
 module.exports = new Lambda();

test/main.js

@@ -6,6 +6,7 @@ var lambda = require('../lib/main');
 var fs = require('fs');
 var _ = require('lodash');
 var zip = require('node-zip');
+var rimraf = require('rimraf');
 
 var assert = chai.assert;
 
@@ -23,7 +24,8 @@ var originalProgram = {
   runtime: 'nodejs',
   region: 'us-east-1,us-west-2,eu-west-1',
   eventFile: 'event.json',
-  contextFile: 'context.json'
+  contextFile: 'context.json',
+  prebuiltDirectory: '',
 };
 
 var codeDirectory = lambda._codeDirectory(program);
@@ -87,7 +89,7 @@ describe('node-lambda', function () {
     });
 
     it('rsync an index.js as well as other files', function (done) {
-      lambda._rsync(program, codeDirectory, function (err, result) {
+      lambda._rsync(program, '.', codeDirectory, function (err, result) {
         var contents = fs.readdirSync(codeDirectory);
 
         result = _.includes(contents, 'index.js') ||
@@ -105,7 +107,7 @@ describe('node-lambda', function () {
       });
 
       it('rsync an index.js as well as other files', function (done) {
-        lambda._rsync(program, codeDirectory, function (err, result) {
+        lambda._rsync(program, '.', codeDirectory, function (err, result) {
           var contents = fs.readdirSync(codeDirectory);
 
           result = _.includes(contents, 'index.js') ||
@@ -117,7 +119,7 @@ describe('node-lambda', function () {
       });
 
       it('rsync excludes files matching excludeGlobs', function (done) {
-        lambda._rsync(program, codeDirectory, function (err, result) {
+        lambda._rsync(program, '.', codeDirectory, function (err, result) {
           var contents = fs.readdirSync(codeDirectory);
 
           result = _.includes(contents, 'node-lambda.png') ||
@@ -137,7 +139,7 @@ describe('node-lambda', function () {
           return done(err);
         }
 
-        lambda._rsync(program, codeDirectory, function (err) {
+        lambda._rsync(program, '.', codeDirectory, function (err) {
           if (err) {
             return done(err);
           }
@@ -168,7 +170,7 @@ describe('node-lambda', function () {
           return done(err);
         }
 
-        lambda._rsync(program, codeDirectory, function (err) {
+        lambda._rsync(program, '.', codeDirectory, function (err) {
           if (err) {
             return done(err);
           }
@@ -213,7 +215,33 @@ describe('node-lambda', function () {
         assert.equal(result, true);
         done();
       })
-    })
+    });
+
+    it('packages a prebuilt module without installing', function (done) {
+      var path = '.build_' + Date.now();
+      after(function() {
+        rimraf.sync(path, fs);
+      });
+
+      fs.mkdirSync(path);
+      fs.mkdirSync(path + '/d');
+      fs.writeFileSync(path + '/testa', 'a');
+      fs.writeFileSync(path + '/d/testb', 'b');
+
+      program.prebuiltDirectory = path;
+      lambda._archive(program, function (err, data) {
+        var archive = new zip(data);
+        var contents = _.map(archive.files, function (f) {
+          return f.name.toString();
+        });
+        var result = _.includes(contents, 'testa');
+        assert.equal(result, true);
+        result = _.includes(contents, 'd/testb');
+        assert.equal(result, true);
+        done();
+
+      })
+    });
   });
 
   describe('environment variable injection', function () {