Merge pull request #326 from aws/develop

release: 1.11.0

Author: mndeveci

.appveyor.yml

@@ -6,7 +6,7 @@ image:
 environment:
   GOVERSION: 1.11
   GRADLE_OPTS: -Dorg.gradle.daemon=false
-  nodejs_version: "10.10.0"
+  nodejs_version: "14.17.6"
 
   matrix:
     - PYTHON: "C:\\Python36-x64"
@@ -93,7 +93,7 @@ for:
       - sh: "source ${HOME}/venv${PYTHON_VERSION}/bin/activate"
       - sh: "rvm use 2.5"
       - sh: "nvm install ${nodejs_version}"
-      - sh: "npm install npm@5.6.0 -g"
+      - sh: "npm install npm@7.24.2 -g"
       - sh: "npm -v"
       - sh: "echo $PATH"
       - sh: "java --version"

Makefile

@@ -4,7 +4,7 @@ init:
 test:
 	# Run unit tests
 	# Fail if coverage falls below 94%
-	LAMBDA_BUILDERS_DEV=1 pytest --cov aws_lambda_builders --cov-report term-missing --cov-fail-under 94 tests/unit tests/functional
+	LAMBDA_BUILDERS_DEV=1 pytest -vv --cov aws_lambda_builders --cov-report term-missing --cov-fail-under 94 tests/unit tests/functional
 
 func-test:
 	LAMBDA_BUILDERS_DEV=1 pytest tests/functional

aws_lambda_builders/__init__.py

@@ -1,5 +1,5 @@
 """
 AWS Lambda Builder Library
 """
-__version__ = "1.10.0"
+__version__ = "1.11.0"
 RPC_PROTOCOL_VERSION = "0.3"

aws_lambda_builders/builder.py

@@ -69,7 +69,10 @@ def build(
         dependencies_dir=None,
         combine_dependencies=True,
         architecture=X86_64,
+        is_building_layer=False,
+        experimental_flags=None,
     ):
+        # pylint: disable-msg=too-many-locals
         """
         Actually build the code by running workflows
 
@@ -127,6 +130,14 @@ def build(
         :type architecture: str
         :param architecture:
             Type of architecture x86_64 and arm64 for Lambda Function
+
+        :type is_building_layer: bool
+        :param is_building_layer:
+            Boolean flag which will be set True if current build operation is being executed for layers
+
+        :type experimental_flags: list
+        :param experimental_flags:
+            List of strings, which will indicate enabled experimental flags for the current build session
         """
 
         if not os.path.exists(scratch_dir):
@@ -146,6 +157,8 @@ def build(
             dependencies_dir=dependencies_dir,
             combine_dependencies=combine_dependencies,
             architecture=architecture,
+            is_building_layer=is_building_layer,
+            experimental_flags=experimental_flags,
         )
 
         return workflow.run()

aws_lambda_builders/utils.py

@@ -12,7 +12,7 @@
 LOG = logging.getLogger(__name__)
 
 
-def copytree(source, destination, ignore=None):
+def copytree(source, destination, ignore=None, include=None):
     """
     Similar to shutil.copytree except that it removes the limitation that the destination directory should
     be present.
@@ -29,17 +29,25 @@ def copytree(source, destination, ignore=None):
     :param ignore:
         A function that returns a set of file names to ignore, given a list of available file names. Similar to the
         ``ignore`` property of ``shutils.copytree`` method
+
+    :type include: Callable[[str], bool]
+    :param include:
+        A function that will decide whether a file should be copied or skipped it. It accepts file name as parameter
+        and return True or False. Returning True will continue copy operation, returning False will skip copy operation
+        for that file
     """
 
     if not os.path.exists(source):
         LOG.warning("Skipping copy operation since source %s does not exist", source)
         return
 
     if not os.path.exists(destination):
+        LOG.debug("Creating target folders at %s", destination)
         os.makedirs(destination)
 
         try:
             # Let's try to copy the directory metadata from source to destination
+            LOG.debug("Copying directory metadata from source (%s) to destination (%s)", source, destination)
             shutil.copystat(source, destination)
         except OSError as ex:
             # Can't copy file access times in Windows
@@ -54,14 +62,20 @@ def copytree(source, destination, ignore=None):
     for name in names:
         # Skip ignored names
         if name in ignored_names:
+            LOG.debug("File (%s) is in ignored set, skipping it", name)
             continue
 
         new_source = os.path.join(source, name)
         new_destination = os.path.join(destination, name)
 
+        if include and not os.path.isdir(new_source) and not include(name):
+            LOG.debug("File (%s) doesn't satisfy the include rule, skipping it", name)
+            continue
+
         if os.path.isdir(new_source):
-            copytree(new_source, new_destination, ignore=ignore)
+            copytree(new_source, new_destination, ignore=ignore, include=include)
         else:
+            LOG.debug("Copying source file (%s) to destination (%s)", new_source, new_destination)
             shutil.copy2(new_source, new_destination)
 
 

aws_lambda_builders/workflow.py

@@ -164,7 +164,10 @@ def __init__(
         dependencies_dir=None,
         combine_dependencies=True,
         architecture=X86_64,
+        is_building_layer=False,
+        experimental_flags=None,
     ):
+        # pylint: disable-msg=too-many-locals
         """
         Initialize the builder with given arguments. These arguments together form the "public API" that each
         build action must support at the minimum.
@@ -200,6 +203,12 @@ def __init__(
             from dependency_folder into build folder
         architecture : str, optional
             Architecture type either arm64 or x86_64 for which the build will be based on in AWS lambda, by default X86_64
+
+        is_building_layer: bool, optional
+            Boolean flag which will be set True if current build operation is being executed for layers
+
+        experimental_flags: list, optional
+            List of strings, which will indicate enabled experimental flags for the current build session
         """
 
         self.source_dir = source_dir
@@ -215,6 +224,8 @@ def __init__(
         self.dependencies_dir = dependencies_dir
         self.combine_dependencies = combine_dependencies
         self.architecture = architecture
+        self.is_building_layer = is_building_layer
+        self.experimental_flags = experimental_flags if experimental_flags else []
 
         # Actions are registered by the subclasses as they seem fit
         self.actions = []

aws_lambda_builders/workflows/__init__.py

@@ -11,3 +11,4 @@
 import aws_lambda_builders.workflows.java_maven
 import aws_lambda_builders.workflows.dotnet_clipackage
 import aws_lambda_builders.workflows.custom_make
+import aws_lambda_builders.workflows.nodejs_npm_esbuild

aws_lambda_builders/workflows/java/utils.py

@@ -6,7 +6,10 @@
 import platform
 import shutil
 import subprocess
-from aws_lambda_builders.utils import which
+from aws_lambda_builders.utils import which, copytree
+
+
+EXPERIMENTAL_MAVEN_SCOPE_AND_LAYER_FLAG = "experimentalMavenScopeAndLayer"
 
 
 class OSUtils(object):
@@ -37,17 +40,8 @@ def exists(self, p):
     def which(self, executable, executable_search_paths=None):
         return which(executable, executable_search_paths=executable_search_paths)
 
-    def copytree(self, source, destination):
-        if not os.path.exists(destination):
-            self.makedirs(destination)
-        names = self.listdir(source)
-        for name in names:
-            new_source = os.path.join(source, name)
-            new_destination = os.path.join(destination, name)
-            if os.path.isdir(new_source):
-                self.copytree(new_source, new_destination)
-            else:
-                self.copy(new_source, new_destination)
+    def copytree(self, source, destination, ignore=None, include=None):
+        copytree(source, destination, ignore=ignore, include=include)
 
     def makedirs(self, d):
         return os.makedirs(d)
@@ -58,3 +52,21 @@ def rmtree(self, d):
     @property
     def pipe(self):
         return subprocess.PIPE
+
+
+def jar_file_filter(file_name):
+    """
+    A function that will filter .jar files for copy operation
+
+    :type file_name: str
+    :param file_name:
+        Name of the file that will be checked against if it ends with .jar or not
+    """
+    return bool(file_name) and isinstance(file_name, str) and file_name.endswith(".jar")
+
+
+def is_experimental_maven_scope_and_layers_active(experimental_flags):
+    """
+    A function which will determine if experimental maven scope and layer changes are active
+    """
+    return bool(experimental_flags) and EXPERIMENTAL_MAVEN_SCOPE_AND_LAYER_FLAG in experimental_flags

aws_lambda_builders/workflows/java_gradle/actions.py

@@ -5,6 +5,7 @@
 import os
 from aws_lambda_builders.actions import ActionFailedError, BaseAction, Purpose
 from .gradle import GradleExecutionError
+from ..java.utils import jar_file_filter
 
 
 class JavaGradleBuildAction(BaseAction):
@@ -56,7 +57,7 @@ def _build_project(self, init_script_file):
 
 
 class JavaGradleCopyArtifactsAction(BaseAction):
-    NAME = "CopyArtifacts"
+    NAME = "JavaGradleCopyArtifacts"
     DESCRIPTION = "Copying the built artifacts"
     PURPOSE = Purpose.COPY_SOURCE
 
@@ -77,3 +78,26 @@ def _copy_artifacts(self):
             self.os_utils.copytree(lambda_build_output, self.artifacts_dir)
         except Exception as ex:
             raise ActionFailedError(str(ex))
+
+
+class JavaGradleCopyLayerArtifactsAction(JavaGradleCopyArtifactsAction):
+    """
+    Java layers does not support using .class files in it.
+    This action (different from the parent one) copies contents of the layer as jar files and place it
+    into the artifact folder
+    """
+
+    NAME = "JavaGradleCopyLayerArtifacts"
+
+    def _copy_artifacts(self):
+        lambda_build_output = os.path.join(self.build_dir, "build", "libs")
+        layer_dependencies = os.path.join(self.build_dir, "build", "distributions", "lambda-build", "lib")
+        try:
+            if not self.os_utils.exists(self.artifacts_dir):
+                self.os_utils.makedirs(self.artifacts_dir)
+            self.os_utils.copytree(
+                lambda_build_output, os.path.join(self.artifacts_dir, "lib"), include=jar_file_filter
+            )
+            self.os_utils.copytree(layer_dependencies, os.path.join(self.artifacts_dir, "lib"), include=jar_file_filter)
+        except Exception as ex:
+            raise ActionFailedError(str(ex))

aws_lambda_builders/workflows/java_gradle/workflow.py

@@ -6,9 +6,9 @@
 from aws_lambda_builders.actions import CleanUpAction
 from aws_lambda_builders.workflow import BaseWorkflow, Capability
 from aws_lambda_builders.workflows.java.actions import JavaCopyDependenciesAction, JavaMoveDependenciesAction
-from aws_lambda_builders.workflows.java.utils import OSUtils
+from aws_lambda_builders.workflows.java.utils import OSUtils, is_experimental_maven_scope_and_layers_active
 
-from .actions import JavaGradleBuildAction, JavaGradleCopyArtifactsAction
+from .actions import JavaGradleBuildAction, JavaGradleCopyArtifactsAction, JavaGradleCopyLayerArtifactsAction
 from .gradle import SubprocessGradle
 from .gradle_resolver import GradleResolver
 from .gradle_validator import GradleValidator
@@ -33,9 +33,16 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, **kwar
 
         subprocess_gradle = SubprocessGradle(gradle_binary=self.binaries["gradle"], os_utils=self.os_utils)
 
+        copy_artifacts_action = JavaGradleCopyArtifactsAction(
+            source_dir, artifacts_dir, self.build_output_dir, self.os_utils
+        )
+        if self.is_building_layer and is_experimental_maven_scope_and_layers_active(self.experimental_flags):
+            copy_artifacts_action = JavaGradleCopyLayerArtifactsAction(
+                source_dir, artifacts_dir, self.build_output_dir, self.os_utils
+            )
         self.actions = [
             JavaGradleBuildAction(source_dir, manifest_path, subprocess_gradle, scratch_dir, self.os_utils),
-            JavaGradleCopyArtifactsAction(source_dir, artifacts_dir, self.build_output_dir, self.os_utils),
+            copy_artifacts_action,
         ]
 
         if self.dependencies_dir: