Merge pull request #332 from aws/develop

chore: Merge develop into master

Author: mildaniel

aws_lambda_builders/__init__.py

@@ -1,5 +1,5 @@
 """
 AWS Lambda Builder Library
 """
-__version__ = "1.11.0"
+__version__ = "1.12.0"
 RPC_PROTOCOL_VERSION = "0.3"

aws_lambda_builders/__main__.py

@@ -129,6 +129,8 @@ def main():  # pylint: disable=too-many-statements
             dependencies_dir=params.get("dependencies_dir", None),
             combine_dependencies=params.get("combine_dependencies", True),
             architecture=params.get("architecture", X86_64),
+            is_building_layer=params.get("is_building_layer", False),
+            experimental_flags=params.get("experimental_flags", []),
         )
 
         # Return a success response

aws_lambda_builders/actions.py

@@ -136,7 +136,7 @@ def execute(self):
             if os.path.isdir(dependencies_source):
                 copytree(dependencies_source, new_destination)
             else:
-                os.makedirs(os.path.dirname(dependencies_source), exist_ok=True)
+                os.makedirs(os.path.dirname(new_destination), exist_ok=True)
                 shutil.copy2(dependencies_source, new_destination)
 
 
@@ -162,6 +162,10 @@ def execute(self):
             dependencies_source = os.path.join(self.artifact_dir, name)
             new_destination = os.path.join(self.dest_dir, name)
 
+            # shutil.move can't create subfolders if this is the first file in that folder
+            if os.path.isfile(dependencies_source):
+                os.makedirs(os.path.dirname(new_destination), exist_ok=True)
+
             shutil.move(dependencies_source, new_destination)
 
 

aws_lambda_builders/workflows/nodejs_npm_esbuild/actions.py

@@ -2,6 +2,8 @@
 Actions specific to the esbuild bundler
 """
 import logging
+from tempfile import NamedTemporaryFile
+
 from pathlib import Path
 
 from aws_lambda_builders.actions import BaseAction, Purpose, ActionFailedError
@@ -23,7 +25,16 @@ class EsbuildBundleAction(BaseAction):
 
     ENTRY_POINTS = "entry_points"
 
-    def __init__(self, scratch_dir, artifacts_dir, bundler_config, osutils, subprocess_esbuild):
+    def __init__(
+        self,
+        scratch_dir,
+        artifacts_dir,
+        bundler_config,
+        osutils,
+        subprocess_esbuild,
+        subprocess_nodejs=None,
+        skip_deps=False,
+    ):
         """
         :type scratch_dir: str
         :param scratch_dir: an existing (writable) directory for temporary files
@@ -35,15 +46,23 @@ def __init__(self, scratch_dir, artifacts_dir, bundler_config, osutils, subproce
         :type osutils: aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils
         :param osutils: An instance of OS Utilities for file manipulation
 
-        :type subprocess_esbuild: aws_lambda_builders.workflows.nodejs_npm.npm.SubprocessEsbuild
+        :type subprocess_esbuild: aws_lambda_builders.workflows.nodejs_npm_esbuild.esbuild.SubprocessEsbuild
         :param subprocess_esbuild: An instance of the Esbuild process wrapper
+
+        :type subprocess_nodejs: aws_lambda_builders.workflows.nodejs_npm_esbuild.node.SubprocessNodejs
+        :param subprocess_nodejs: An instance of the nodejs process wrapper
+
+        :type skip_deps: bool
+        :param skip_deps: if dependencies should be omitted from bundling
         """
         super(EsbuildBundleAction, self).__init__()
         self.scratch_dir = scratch_dir
         self.artifacts_dir = artifacts_dir
         self.bundler_config = bundler_config
         self.osutils = osutils
         self.subprocess_esbuild = subprocess_esbuild
+        self.skip_deps = skip_deps
+        self.subprocess_nodejs = subprocess_nodejs
 
     def execute(self):
         """
@@ -81,11 +100,73 @@ def execute(self):
             args.append("--sourcemap")
         args.append("--target={}".format(target))
         args.append("--outdir={}".format(self.artifacts_dir))
+
+        if self.skip_deps:
+            LOG.info("Running custom esbuild using Node.js")
+            script = EsbuildBundleAction._get_node_esbuild_template(
+                explicit_entry_points, target, self.artifacts_dir, minify, sourcemap
+            )
+            self._run_external_esbuild_in_nodejs(script)
+            return
+
         try:
             self.subprocess_esbuild.run(args, cwd=self.scratch_dir)
         except EsbuildExecutionError as ex:
             raise ActionFailedError(str(ex))
 
+    def _run_external_esbuild_in_nodejs(self, script):
+        """
+        Run esbuild in a separate process through Node.js
+        Workaround for https://github.com/evanw/esbuild/issues/1958
+
+        :type script: str
+        :param script: Node.js script to execute
+
+        :raises lambda_builders.actions.ActionFailedError: when esbuild packaging fails
+        """
+        with NamedTemporaryFile(dir=self.scratch_dir, mode="w") as tmp:
+            tmp.write(script)
+            tmp.flush()
+            try:
+                self.subprocess_nodejs.run([tmp.name], cwd=self.scratch_dir)
+            except EsbuildExecutionError as ex:
+                raise ActionFailedError(str(ex))
+
+    @staticmethod
+    def _get_node_esbuild_template(entry_points, target, out_dir, minify, sourcemap):
+        """
+        Get the esbuild nodejs plugin template
+
+        :type entry_points: List[str]
+        :param entry_points: list of entry points
+
+        :type target: str
+        :param target: target version
+
+        :type out_dir: str
+        :param out_dir: output directory to bundle into
+
+        :type minify: bool
+        :param minify: if bundled code should be minified
+
+        :type sourcemap: bool
+        :param sourcemap: if esbuild should produce a sourcemap
+
+        :rtype: str
+        :return: formatted template
+        """
+        curr_dir = Path(__file__).resolve().parent
+        with open(str(Path(curr_dir, "esbuild-plugin.js.template")), "r") as f:
+            input_str = f.read()
+            result = input_str.format(
+                target=target,
+                minify="true" if minify else "false",
+                sourcemap="true" if sourcemap else "false",
+                out_dir=repr(out_dir),
+                entry_points=entry_points,
+            )
+        return result
+
     def _get_explicit_file_type(self, entry_point, entry_path):
         """
         Get an entry point with an explicit .ts or .js suffix.
@@ -112,3 +193,67 @@ def _get_explicit_file_type(self, entry_point, entry_path):
                 return entry_point + ext
 
         raise ActionFailedError("entry point {} does not exist".format(entry_path))
+
+
+class EsbuildCheckVersionAction(BaseAction):
+    """
+    A Lambda Builder Action that verifies that esbuild is a version supported by sam accelerate
+    """
+
+    NAME = "EsbuildCheckVersion"
+    DESCRIPTION = "Checking esbuild version"
+    PURPOSE = Purpose.COMPILE_SOURCE
+
+    MIN_VERSION = "0.14.13"
+
+    def __init__(self, scratch_dir, subprocess_esbuild):
+        """
+        :type scratch_dir: str
+        :param scratch_dir: temporary directory where esbuild is executed
+
+        :type subprocess_esbuild: aws_lambda_builders.workflows.nodejs_npm_esbuild.esbuild.SubprocessEsbuild
+        :param subprocess_esbuild: An instance of the Esbuild process wrapper
+        """
+        super().__init__()
+        self.scratch_dir = scratch_dir
+        self.subprocess_esbuild = subprocess_esbuild
+
+    def execute(self):
+        """
+        Runs the action.
+
+        :raises lambda_builders.actions.ActionFailedError: when esbuild version checking fails
+        """
+        args = ["--version"]
+
+        try:
+            version = self.subprocess_esbuild.run(args, cwd=self.scratch_dir)
+        except EsbuildExecutionError as ex:
+            raise ActionFailedError(str(ex))
+
+        LOG.debug("Found esbuild with version: %s", version)
+
+        try:
+            check_version = EsbuildCheckVersionAction._get_version_tuple(self.MIN_VERSION)
+            esbuild_version = EsbuildCheckVersionAction._get_version_tuple(version)
+
+            if esbuild_version < check_version:
+                raise ActionFailedError(
+                    f"Unsupported esbuild version. To use a dependency layer, the esbuild version must be at "
+                    f"least {self.MIN_VERSION}. Version found: {version}"
+                )
+        except (TypeError, ValueError) as ex:
+            raise ActionFailedError(f"Unable to parse esbuild version: {str(ex)}")
+
+    @staticmethod
+    def _get_version_tuple(version_string):
+        """
+        Get an integer tuple representation of the version for comparison
+
+        :type version_string: str
+        :param version_string: string containing the esbuild version
+
+        :rtype: tuple
+        :return: version tuple used for comparison
+        """
+        return tuple(map(int, version_string.split(".")))

aws_lambda_builders/workflows/nodejs_npm_esbuild/esbuild-plugin.js.template

@@ -0,0 +1,19 @@
+let skipBundleNodeModules = {{
+  name: 'make-all-packages-external',
+  setup(build) {{
+    let filter = /^[^.\/]|^\.[^.\/]|^\.\.[^\/]/ // Must not start with "/" or "./" or "../"
+    build.onResolve({{ filter }}, args => ({{ path: args.path, external: true }}))
+  }},
+}}
+
+require('esbuild').build({{
+  entryPoints: {entry_points},
+  bundle: true,
+  platform: 'node',
+  format: 'cjs',
+  target: '{target}',
+  sourcemap: {sourcemap},
+  outdir: {out_dir},
+  minify: {minify},
+  plugins: [skipBundleNodeModules],
+}}).catch(() => process.exit(1))

aws_lambda_builders/workflows/nodejs_npm_esbuild/node.py

@@ -0,0 +1,104 @@
+"""
+Wrapper around calling nodejs through a subprocess.
+"""
+
+import logging
+
+from aws_lambda_builders.exceptions import LambdaBuilderError
+
+LOG = logging.getLogger(__name__)
+
+
+class NodejsExecutionError(LambdaBuilderError):
+
+    """
+    Exception raised in case nodejs execution fails.
+    It will pass on the standard error output from the Node.js console.
+    """
+
+    MESSAGE = "Nodejs Failed: {message}"
+
+
+class SubprocessNodejs(object):
+
+    """
+    Wrapper around the nodejs command line utility, making it
+    easy to consume execution results.
+    """
+
+    def __init__(self, osutils, executable_search_paths, which):
+        """
+        :type osutils: aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils
+        :param osutils: An instance of OS Utilities for file manipulation
+
+        :type executable_search_paths: list
+        :param executable_search_paths: List of paths to the node package binary utilities. This will
+            be used to find embedded Nodejs at runtime if present in the package
+
+        :type which: aws_lambda_builders.utils.which
+        :param which: Function to get paths which conform to the given mode on the PATH
+            with the prepended additional search paths
+        """
+        self.osutils = osutils
+        self.executable_search_paths = executable_search_paths
+        self.which = which
+
+    def nodejs_binary(self):
+        """
+        Finds the Nodejs binary at runtime.
+
+        The utility may be present as a package dependency of the Lambda project,
+        or in the global path. If there is one in the Lambda project, it should
+        be preferred over a global utility. The check has to be executed
+        at runtime, since nodejs dependencies will be installed by the workflow
+        using one of the previous actions.
+        """
+
+        LOG.debug("checking for nodejs in: %s", self.executable_search_paths)
+        binaries = self.which("node", executable_search_paths=self.executable_search_paths)
+        LOG.debug("potential nodejs binaries: %s", binaries)
+
+        if binaries:
+            return binaries[0]
+        else:
+            raise NodejsExecutionError(message="cannot find nodejs")
+
+    def run(self, args, cwd=None):
+
+        """
+        Runs the action.
+
+        :type args: list
+        :param args: Command line arguments to pass to Nodejs
+
+        :type cwd: str
+        :param cwd: Directory where to execute the command (defaults to current dir)
+
+        :rtype: str
+        :return: text of the standard output from the command
+
+        :raises aws_lambda_builders.workflows.nodejs_npm.npm.NodejsExecutionError:
+            when the command executes with a non-zero return code. The exception will
+            contain the text of the standard error output from the command.
+
+        :raises ValueError: if arguments are not provided, or not a list
+        """
+
+        if not isinstance(args, list):
+            raise ValueError("args must be a list")
+
+        if not args:
+            raise ValueError("requires at least one arg")
+
+        invoke_nodejs = [self.nodejs_binary()] + args
+
+        LOG.debug("executing Nodejs: %s", invoke_nodejs)
+
+        p = self.osutils.popen(invoke_nodejs, stdout=self.osutils.pipe, stderr=self.osutils.pipe, cwd=cwd)
+
+        out, err = p.communicate()
+
+        if p.returncode != 0:
+            raise NodejsExecutionError(message=err.decode("utf8").strip())
+
+        return out.decode("utf8").strip()