Merge pull request #381 from aws/develop

aws-lambda-builders release 1.19.0

Author: mildaniel

aws_lambda_builders/__init__.py

@@ -1,5 +1,5 @@
 """
 AWS Lambda Builder Library
 """
-__version__ = "1.18.0"
+__version__ = "1.19.0"
 RPC_PROTOCOL_VERSION = "0.3"

aws_lambda_builders/actions.py

@@ -5,8 +5,10 @@
 import logging
 import os
 import shutil
+from pathlib import Path
 from typing import Set, Iterator, Tuple
 
+from aws_lambda_builders import utils
 from aws_lambda_builders.utils import copytree
 
 LOG = logging.getLogger(__name__)
@@ -31,6 +33,9 @@ class Purpose(object):
     # Action is copying source code
     COPY_SOURCE = "COPY_SOURCE"
 
+    # Action is linking source code
+    LINK_SOURCE = "LINK_SOURCE"
+
     # Action is copying dependencies
     COPY_DEPENDENCIES = "COPY_DEPENDENCIES"
 
@@ -111,6 +116,31 @@ def execute(self):
         copytree(self.source_dir, self.dest_dir, ignore=shutil.ignore_patterns(*self.excludes))
 
 
+class LinkSourceAction(BaseAction):
+
+    NAME = "LinkSource"
+
+    DESCRIPTION = "Linking source code to the target folder"
+
+    PURPOSE = Purpose.LINK_SOURCE
+
+    def __init__(self, source_dir, dest_dir):
+        self._source_dir = source_dir
+        self._dest_dir = dest_dir
+
+    def execute(self):
+        source_files = set(os.listdir(self._source_dir))
+
+        for source_file in source_files:
+            source_path = Path(self._source_dir, source_file)
+            destination_path = Path(self._dest_dir, source_file)
+            if destination_path.exists():
+                os.remove(destination_path)
+            else:
+                os.makedirs(destination_path.parent, exist_ok=True)
+            utils.create_symlink_or_copy(str(source_path), str(destination_path))
+
+
 class CopyDependenciesAction(BaseAction):
 
     NAME = "CopyDependencies"
@@ -175,10 +205,10 @@ def __init__(self, target_dir):
 
     def execute(self):
         if not os.path.isdir(self.target_dir):
-            LOG.info("Clean up action: %s does not exist and will be skipped.", str(self.target_dir))
+            LOG.debug("Clean up action: %s does not exist and will be skipped.", str(self.target_dir))
             return
         targets = os.listdir(self.target_dir)
-        LOG.info("Clean up action: folder %s will be cleaned", str(self.target_dir))
+        LOG.debug("Clean up action: folder %s will be cleaned", str(self.target_dir))
 
         for name in targets:
             target_path = os.path.join(self.target_dir, name)

aws_lambda_builders/exceptions.py

@@ -19,7 +19,7 @@ class MisMatchRuntimeError(LambdaBuilderError):
     MESSAGE = (
         "{language} executable found in your path does not "
         "match runtime. "
-        "\n Expected version: {required_runtime}, Found version: {runtime_path}. "
+        "\n Expected version: {required_runtime}, Found a different version at {runtime_path}. "
         "\n Possibly related: https://github.com/awslabs/aws-lambda-builders/issues/30"
     )
 

aws_lambda_builders/path_resolver.py

@@ -6,10 +6,14 @@
 
 
 class PathResolver(object):
-    def __init__(self, binary, runtime, executable_search_paths=None):
+    def __init__(self, binary, runtime, additional_binaries=None, executable_search_paths=None):
         self.binary = binary
         self.runtime = runtime
         self.executables = [self.runtime, self.binary]
+        self.additional_binaries = additional_binaries
+        if isinstance(additional_binaries, list):
+            self.executables = self.executables + self.additional_binaries
+
         self.executable_search_paths = executable_search_paths
 
     def _which(self):

aws_lambda_builders/utils.py

@@ -6,6 +6,7 @@
 import sys
 import os
 import logging
+from pathlib import Path
 
 from aws_lambda_builders.architecture import X86_64, ARM64
 
@@ -182,3 +183,16 @@ def get_goarch(architecture):
         returns a valid GO Architecture value
     """
     return "arm64" if architecture == ARM64 else "amd64"
+
+
+def create_symlink_or_copy(source: str, destination: str) -> None:
+    """Tries to create symlink, if it fails it will copy source into destination"""
+    LOG.debug("Creating symlink; source: %s, destination: %s", source, destination)
+    try:
+        os.symlink(Path(source).absolute(), Path(destination).absolute())
+    except OSError as ex:
+        LOG.warning(
+            "Symlink operation is failed, falling back to copying files",
+            exc_info=ex if LOG.isEnabledFor(logging.DEBUG) else None,
+        )
+        copytree(source, destination)

aws_lambda_builders/workflows/custom_make/actions.py

@@ -22,7 +22,16 @@ class CustomMakeAction(BaseAction):
     DESCRIPTION = "Running build target on Makefile"
     PURPOSE = Purpose.COMPILE_SOURCE
 
-    def __init__(self, artifacts_dir, scratch_dir, manifest_path, osutils, subprocess_make, build_logical_id):
+    def __init__(
+        self,
+        artifacts_dir,
+        scratch_dir,
+        manifest_path,
+        osutils,
+        subprocess_make,
+        build_logical_id,
+        working_directory=None,
+    ):
         """
         :type artifacts_dir: str
         :param artifacts_dir: directory where artifacts needs to be stored.
@@ -38,6 +47,13 @@ def __init__(self, artifacts_dir, scratch_dir, manifest_path, osutils, subproces
 
         :type subprocess_make aws_lambda_builders.workflows.custom_make.make.SubprocessMake
         :param subprocess_make: An instance of the Make process wrapper
+
+        :type build_logical_id: str
+        :param build_logical_id: the lambda resource logical id that will be built by the custom action.
+
+        :type working_directory: str
+        :param working_directory: path to the working directory where the Makefile will be executed. Use the scratch_dir
+        as the working directory if the input working_directory is None
         """
         super(CustomMakeAction, self).__init__()
         self.artifacts_dir = artifacts_dir
@@ -46,6 +62,7 @@ def __init__(self, artifacts_dir, scratch_dir, manifest_path, osutils, subproces
         self.osutils = osutils
         self.subprocess_make = subprocess_make
         self.build_logical_id = build_logical_id
+        self.working_directory = working_directory if working_directory else scratch_dir
 
     @property
     def artifact_dir_path(self):
@@ -91,7 +108,7 @@ def execute(self):
                     "build-{logical_id}".format(logical_id=self.build_logical_id),
                 ],
                 env=current_env,
-                cwd=self.scratch_dir,
+                cwd=self.working_directory,
             )
         except MakeExecutionError as ex:
             raise ActionFailedError(str(ex))

aws_lambda_builders/workflows/custom_make/workflow.py

@@ -34,6 +34,7 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim
         # Find the logical id of the function to be built.
         options = kwargs.get("options") or {}
         build_logical_id = options.get("build_logical_id", None)
+        working_directory = options.get("working_directory", scratch_dir)
 
         if not build_logical_id:
             raise WorkflowFailedError(
@@ -51,6 +52,7 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim
             osutils=self.os_utils,
             subprocess_make=subprocess_make,
             build_logical_id=build_logical_id,
+            working_directory=working_directory,
         )
 
         self.actions = [CopySourceAction(source_dir, scratch_dir, excludes=self.EXCLUDED_FILES), make_action]

aws_lambda_builders/workflows/nodejs_npm/actions.py

@@ -80,7 +80,7 @@ class NodejsNpmInstallAction(BaseAction):
     DESCRIPTION = "Installing dependencies from NPM"
     PURPOSE = Purpose.RESOLVE_DEPENDENCIES
 
-    def __init__(self, artifacts_dir, subprocess_npm, is_production=True):
+    def __init__(self, artifacts_dir, subprocess_npm):
         """
         :type artifacts_dir: str
         :param artifacts_dir: an existing (writable) directory with project source files.
@@ -96,22 +96,18 @@ def __init__(self, artifacts_dir, subprocess_npm, is_production=True):
         super(NodejsNpmInstallAction, self).__init__()
         self.artifacts_dir = artifacts_dir
         self.subprocess_npm = subprocess_npm
-        self.is_production = is_production
 
     def execute(self):
         """
         Runs the action.
 
         :raises lambda_builders.actions.ActionFailedError: when NPM execution fails
         """
-
-        mode = "--production" if self.is_production else "--production=false"
-
         try:
             LOG.debug("NODEJS installing in: %s", self.artifacts_dir)
 
             self.subprocess_npm.run(
-                ["install", "-q", "--no-audit", "--no-save", mode, "--unsafe-perm"], cwd=self.artifacts_dir
+                ["install", "-q", "--no-audit", "--no-save", "--unsafe-perm", "--production"], cwd=self.artifacts_dir
             )
 
         except NpmExecutionError as ex:

aws_lambda_builders/workflows/nodejs_npm/workflow.py

@@ -145,7 +145,7 @@ def get_resolvers(self):
         return [PathResolver(runtime=self.runtime, binary="npm")]
 
     @staticmethod
-    def get_install_action(source_dir, artifacts_dir, subprocess_npm, osutils, build_options, is_production=True):
+    def get_install_action(source_dir, artifacts_dir, subprocess_npm, osutils, build_options):
         """
         Get the install action used to install dependencies at artifacts_dir
 
@@ -180,4 +180,4 @@ def get_install_action(source_dir, artifacts_dir, subprocess_npm, osutils, build
         if (osutils.file_exists(lockfile_path) or osutils.file_exists(shrinkwrap_path)) and npm_ci_option:
             return NodejsNpmCIAction(artifacts_dir, subprocess_npm=subprocess_npm)
 
-        return NodejsNpmInstallAction(artifacts_dir, subprocess_npm=subprocess_npm, is_production=is_production)
+        return NodejsNpmInstallAction(artifacts_dir, subprocess_npm=subprocess_npm)

aws_lambda_builders/workflows/nodejs_npm_esbuild/DESIGN.md

@@ -117,24 +117,23 @@ testing flow before invoking `sam build`. For additional typescript caveats with
 
 #### Configuring the bundler
 
-The Lambda builder invokes `esbuild` with sensible defaults that will work for the majority of cases. Importantly, the following three parameters are set by default
+The Lambda builder invokes `esbuild` with sensible defaults that will work for the majority of cases. Importantly, the following parameters are set by default
 
 * `--minify`, as it [produces a smaller runtime package](https://esbuild.github.io/api/#minify)
-* `--sourcemap`, as it generates a [source map that allows for correct stack trace reporting](https://esbuild.github.io/api/#sourcemap) in case of errors (see the [Error reporting](#error-reporting) section above)
 * `--target es2020`, as it allows for javascript features present in Node 14
 
 Users might want to tweak some of these runtime arguments for a specific project, for example not including the source map to further reduce the package size, or restricting javascript features to an older version. The Lambda builder allows this with optional sub-properties of the `aws_sam` configuration property.
 
 * `target`: string, corresponding to a supported [esbuild target](https://esbuild.github.io/api/#target) property
 * `minify`: boolean, defaulting to `true`
-* `sourcemap`: boolean, defaulting to `true`
+* `sourcemap`: boolean, defaulting to `false`
 
-Here is an example that deactivates minification and source maps, and supports JavaScript features compatible with Node.js version 10.
+Here is an example that deactivates minification, enables source maps, and supports JavaScript features compatible with Node.js version 10.
 
 ```json
 {
     "entry_points": ["included.ts"],
     "target": "node10",
     "minify": false,
-    "sourcemap": false
+    "sourcemap": true
 }