Merge pull request #361 from aws/develop

chore: Merge develop to master

Author: hawflau

aws_lambda_builders/__init__.py

@@ -1,5 +1,5 @@
 """
 AWS Lambda Builder Library
 """
-__version__ = "1.15.0"
+__version__ = "1.16.0"
 RPC_PROTOCOL_VERSION = "0.3"

aws_lambda_builders/validator.py

@@ -13,6 +13,7 @@
 SUPPORTED_RUNTIMES = {
     "nodejs12.x": [ARM64, X86_64],
     "nodejs14.x": [ARM64, X86_64],
+    "nodejs16.x": [ARM64, X86_64],
     "python3.6": [X86_64],
     "python3.7": [X86_64],
     "python3.8": [ARM64, X86_64],

aws_lambda_builders/workflows/nodejs_npm/utils.py

@@ -53,3 +53,6 @@ def is_windows(self):
     def parse_json(self, path):
         with open(path) as json_file:
             return json.load(json_file)
+
+    def check_output(self, path):
+        return subprocess.check_output(["node", path])

aws_lambda_builders/workflows/nodejs_npm_esbuild/actions.py

@@ -54,6 +54,9 @@ def __init__(
 
         :type skip_deps: bool
         :param skip_deps: if dependencies should be omitted from bundling
+
+        :type bundler_config: Dict[str,Any]
+        :param bundler_config: the bundler configuration
         """
         super(EsbuildBundleAction, self).__init__()
         self.scratch_dir = scratch_dir
@@ -71,38 +74,29 @@ def execute(self):
         :raises lambda_builders.actions.ActionFailedError: when esbuild packaging fails
         """
 
-        if self.ENTRY_POINTS not in self.bundler_config:
-            raise ActionFailedError(f"{self.ENTRY_POINTS} not set ({self.bundler_config})")
-
-        entry_points = self.bundler_config[self.ENTRY_POINTS]
-
-        if not isinstance(entry_points, list):
-            raise ActionFailedError(f"{self.ENTRY_POINTS} must be a list ({self.bundler_config})")
-
-        if not entry_points:
-            raise ActionFailedError(f"{self.ENTRY_POINTS} must not be empty ({self.bundler_config})")
-
-        entry_paths = [self.osutils.joinpath(self.scratch_dir, entry_point) for entry_point in entry_points]
-
-        LOG.debug("NODEJS building %s using esbuild to %s", entry_paths, self.artifacts_dir)
-
-        explicit_entry_points = []
-        for entry_path, entry_point in zip(entry_paths, entry_points):
-            explicit_entry_points.append(self._get_explicit_file_type(entry_point, entry_path))
+        explicit_entry_points = self._construct_esbuild_entry_points()
 
         args = explicit_entry_points + ["--bundle", "--platform=node", "--format=cjs"]
         minify = self.bundler_config.get("minify", True)
         sourcemap = self.bundler_config.get("sourcemap", True)
         target = self.bundler_config.get("target", "es2020")
+        external = self.bundler_config.get("external", [])
+        loader = self.bundler_config.get("loader", [])
         if minify:
             args.append("--minify")
         if sourcemap:
             args.append("--sourcemap")
+        if external:
+            args.extend(map(lambda x: f"--external:{x}", external))
+        if loader:
+            args.extend(map(lambda x: f"--loader:{x}", loader))
+
         args.append("--target={}".format(target))
         args.append("--outdir={}".format(self.artifacts_dir))
 
         if self.skip_deps:
             LOG.info("Running custom esbuild using Node.js")
+            # Don't pass externals because the esbuild.js template makes everything external
             script = EsbuildBundleAction._get_node_esbuild_template(
                 explicit_entry_points, target, self.artifacts_dir, minify, sourcemap
             )
@@ -132,6 +126,30 @@ def _run_external_esbuild_in_nodejs(self, script):
             except EsbuildExecutionError as ex:
                 raise ActionFailedError(str(ex))
 
+    def _construct_esbuild_entry_points(self):
+        """
+        Construct the list of explicit entry points
+        """
+        if self.ENTRY_POINTS not in self.bundler_config:
+            raise ActionFailedError(f"{self.ENTRY_POINTS} not set ({self.bundler_config})")
+
+        entry_points = self.bundler_config[self.ENTRY_POINTS]
+
+        if not isinstance(entry_points, list):
+            raise ActionFailedError(f"{self.ENTRY_POINTS} must be a list ({self.bundler_config})")
+
+        if not entry_points:
+            raise ActionFailedError(f"{self.ENTRY_POINTS} must not be empty ({self.bundler_config})")
+
+        entry_paths = [self.osutils.joinpath(self.scratch_dir, entry_point) for entry_point in entry_points]
+
+        LOG.debug("NODEJS building %s using esbuild to %s", entry_paths, self.artifacts_dir)
+
+        explicit_entry_points = []
+        for entry_path, entry_point in zip(entry_paths, entry_points):
+            explicit_entry_points.append(self._get_explicit_file_type(entry_point, entry_path))
+        return explicit_entry_points
+
     @staticmethod
     def _get_node_esbuild_template(entry_points, target, out_dir, minify, sourcemap):
         """

tests/integration/workflows/nodejs_npm/test_nodejs_npm.py

@@ -29,29 +29,30 @@ def setUp(self):
         self.no_deps = os.path.join(self.TEST_DATA_FOLDER, "no-deps")
 
         self.builder = LambdaBuilder(language="nodejs", dependency_manager="npm", application_framework=None)
-        self.runtime = "nodejs12.x"
 
     def tearDown(self):
         shutil.rmtree(self.artifacts_dir)
         shutil.rmtree(self.scratch_dir)
         shutil.rmtree(self.dependencies_dir)
 
-    def test_builds_project_without_dependencies(self):
+    @parameterized.expand([("nodejs12.x",), ("nodejs14.x",), ("nodejs16.x",)])
+    def test_builds_project_without_dependencies(self, runtime):
         source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-deps")
 
         self.builder.build(
             source_dir,
             self.artifacts_dir,
             self.scratch_dir,
             os.path.join(source_dir, "package.json"),
-            runtime=self.runtime,
+            runtime=runtime,
         )
 
         expected_files = {"package.json", "included.js"}
         output_files = set(os.listdir(self.artifacts_dir))
         self.assertEqual(expected_files, output_files)
 
-    def test_builds_project_without_manifest(self):
+    @parameterized.expand([("nodejs12.x",), ("nodejs14.x",), ("nodejs16.x",)])
+    def test_builds_project_without_manifest(self, runtime):
         source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-manifest")
 
         with mock.patch.object(logger, "warning") as mock_warning:
@@ -60,38 +61,40 @@ def test_builds_project_without_manifest(self):
                 self.artifacts_dir,
                 self.scratch_dir,
                 os.path.join(source_dir, "package.json"),
-                runtime=self.runtime,
+                runtime=runtime,
             )
 
         expected_files = {"app.js"}
         output_files = set(os.listdir(self.artifacts_dir))
         mock_warning.assert_called_once_with("package.json file not found. Continuing the build without dependencies.")
         self.assertEqual(expected_files, output_files)
 
-    def test_builds_project_and_excludes_hidden_aws_sam(self):
+    @parameterized.expand([("nodejs12.x",), ("nodejs14.x",), ("nodejs16.x",)])
+    def test_builds_project_and_excludes_hidden_aws_sam(self, runtime):
         source_dir = os.path.join(self.TEST_DATA_FOLDER, "excluded-files")
 
         self.builder.build(
             source_dir,
             self.artifacts_dir,
             self.scratch_dir,
             os.path.join(source_dir, "package.json"),
-            runtime=self.runtime,
+            runtime=runtime,
         )
 
         expected_files = {"package.json", "included.js"}
         output_files = set(os.listdir(self.artifacts_dir))
         self.assertEqual(expected_files, output_files)
 
-    def test_builds_project_with_remote_dependencies(self):
+    @parameterized.expand([("nodejs12.x",), ("nodejs14.x",), ("nodejs16.x",)])
+    def test_builds_project_with_remote_dependencies(self, runtime):
         source_dir = os.path.join(self.TEST_DATA_FOLDER, "npm-deps")
 
         self.builder.build(
             source_dir,
             self.artifacts_dir,
             self.scratch_dir,
             os.path.join(source_dir, "package.json"),
-            runtime=self.runtime,
+            runtime=runtime,
         )
 
         expected_files = {"package.json", "included.js", "node_modules"}
@@ -102,15 +105,16 @@ def test_builds_project_with_remote_dependencies(self):
         output_modules = set(os.listdir(os.path.join(self.artifacts_dir, "node_modules")))
         self.assertEqual(expected_modules, output_modules)
 
-    def test_builds_project_with_npmrc(self):
+    @parameterized.expand([("nodejs12.x",), ("nodejs14.x",), ("nodejs16.x",)])
+    def test_builds_project_with_npmrc(self, runtime):
         source_dir = os.path.join(self.TEST_DATA_FOLDER, "npmrc")
 
         self.builder.build(
             source_dir,
             self.artifacts_dir,
             self.scratch_dir,
             os.path.join(source_dir, "package.json"),
-            runtime=self.runtime,
+            runtime=runtime,
         )
 
         expected_files = {"package.json", "included.js", "node_modules"}
@@ -122,8 +126,20 @@ def test_builds_project_with_npmrc(self):
         output_modules = set(os.listdir(os.path.join(self.artifacts_dir, "node_modules")))
         self.assertEqual(expected_modules, output_modules)
 
-    @parameterized.expand(["package-lock", "shrinkwrap", "package-lock-and-shrinkwrap"])
-    def test_builds_project_with_lockfile(self, dir_name):
+    @parameterized.expand(
+        [
+            ("nodejs12.x", "package-lock"),
+            ("nodejs14.x", "package-lock"),
+            ("nodejs16.x", "package-lock"),
+            ("nodejs12.x", "shrinkwrap"),
+            ("nodejs14.x", "shrinkwrap"),
+            ("nodejs16.x", "shrinkwrap"),
+            ("nodejs12.x", "package-lock-and-shrinkwrap"),
+            ("nodejs14.x", "package-lock-and-shrinkwrap"),
+            ("nodejs16.x", "package-lock-and-shrinkwrap"),
+        ]
+    )
+    def test_builds_project_with_lockfile(self, runtime, dir_name):
         expected_files_common = {"package.json", "included.js", "node_modules"}
         expected_files_by_dir_name = {
             "package-lock": {"package-lock.json"},
@@ -138,7 +154,7 @@ def test_builds_project_with_lockfile(self, dir_name):
             self.artifacts_dir,
             self.scratch_dir,
             os.path.join(source_dir, "package.json"),
-            runtime=self.runtime,
+            runtime=runtime,
         )
 
         expected_files = expected_files_common.union(expected_files_by_dir_name[dir_name])
@@ -147,8 +163,8 @@ def test_builds_project_with_lockfile(self, dir_name):
 
         self.assertEqual(expected_files, output_files)
 
-    def test_fails_if_npm_cannot_resolve_dependencies(self):
-
+    @parameterized.expand([("nodejs12.x",), ("nodejs14.x",), ("nodejs16.x",)])
+    def test_fails_if_npm_cannot_resolve_dependencies(self, runtime):
         source_dir = os.path.join(self.TEST_DATA_FOLDER, "broken-deps")
 
         with self.assertRaises(WorkflowFailedError) as ctx:
@@ -157,20 +173,21 @@ def test_fails_if_npm_cannot_resolve_dependencies(self):
                 self.artifacts_dir,
                 self.scratch_dir,
                 os.path.join(source_dir, "package.json"),
-                runtime=self.runtime,
+                runtime=runtime,
             )
 
         self.assertIn("No matching version found for [email protected]", str(ctx.exception))
 
-    def test_builds_project_with_remote_dependencies_without_download_dependencies_with_dependencies_dir(self):
+    @parameterized.expand([("nodejs12.x",), ("nodejs14.x",), ("nodejs16.x",)])
+    def test_builds_project_with_remote_dependencies_without_download_dependencies_with_dependencies_dir(self, runtime):
         source_dir = os.path.join(self.TEST_DATA_FOLDER, "npm-deps")
 
         self.builder.build(
             source_dir,
             self.artifacts_dir,
             self.scratch_dir,
             os.path.join(source_dir, "package.json"),
-            runtime=self.runtime,
+            runtime=runtime,
             dependencies_dir=self.dependencies_dir,
             download_dependencies=False,
         )
@@ -179,15 +196,16 @@ def test_builds_project_with_remote_dependencies_without_download_dependencies_w
         output_files = set(os.listdir(self.artifacts_dir))
         self.assertEqual(expected_files, output_files)
 
-    def test_builds_project_with_remote_dependencies_with_download_dependencies_and_dependencies_dir(self):
+    @parameterized.expand([("nodejs12.x",), ("nodejs14.x",), ("nodejs16.x",)])
+    def test_builds_project_with_remote_dependencies_with_download_dependencies_and_dependencies_dir(self, runtime):
         source_dir = os.path.join(self.TEST_DATA_FOLDER, "npm-deps")
 
         self.builder.build(
             source_dir,
             self.artifacts_dir,
             self.scratch_dir,
             os.path.join(source_dir, "package.json"),
-            runtime=self.runtime,
+            runtime=runtime,
             dependencies_dir=self.dependencies_dir,
             download_dependencies=True,
         )
@@ -208,7 +226,10 @@ def test_builds_project_with_remote_dependencies_with_download_dependencies_and_
         output_dependencies_files = set(os.listdir(os.path.join(self.dependencies_dir)))
         self.assertNotIn(expected_dependencies_files, output_dependencies_files)
 
-    def test_builds_project_with_remote_dependencies_without_download_dependencies_without_dependencies_dir(self):
+    @parameterized.expand([("nodejs12.x",), ("nodejs14.x",), ("nodejs16.x",)])
+    def test_builds_project_with_remote_dependencies_without_download_dependencies_without_dependencies_dir(
+        self, runtime
+    ):
         source_dir = os.path.join(self.TEST_DATA_FOLDER, "npm-deps")
 
         with mock.patch.object(logger, "info") as mock_info:
@@ -217,7 +238,7 @@ def test_builds_project_with_remote_dependencies_without_download_dependencies_w
                 self.artifacts_dir,
                 self.scratch_dir,
                 os.path.join(source_dir, "package.json"),
-                runtime=self.runtime,
+                runtime=runtime,
                 dependencies_dir=None,
                 download_dependencies=False,
             )
@@ -231,15 +252,16 @@ def test_builds_project_with_remote_dependencies_without_download_dependencies_w
             "artifacts directory. "
         )
 
-    def test_builds_project_without_combine_dependencies(self):
+    @parameterized.expand([("nodejs12.x",), ("nodejs14.x",), ("nodejs16.x",)])
+    def test_builds_project_without_combine_dependencies(self, runtime):
         source_dir = os.path.join(self.TEST_DATA_FOLDER, "npm-deps")
 
         self.builder.build(
             source_dir,
             self.artifacts_dir,
             self.scratch_dir,
             os.path.join(source_dir, "package.json"),
-            runtime=self.runtime,
+            runtime=runtime,
             dependencies_dir=self.dependencies_dir,
             download_dependencies=True,
             combine_dependencies=False,