feat: java build changes (#318)

* feat: java build changes related to maven scope and layers
- use "runtime" scope for maven builds if experimental flag is provided
- copy created jar files for layer code instead of classes

* Update the maven DESIGN.md to explain changes.

* add maven integration tests

* add gradle integration tests

* fix formatting & unit tests

* fix issues after merge

* correct dependency with maven tests

* change type from function to Callable

Co-authored-by: Wilton_ <[email protected]>

* disable too-many-locals for workflow.py only

Co-authored-by: Phillip Rower <[email protected]>
Co-authored-by: Wilton_ <[email protected]>

Author: 3 people

Makefile

@@ -4,7 +4,7 @@ init:
 test:
 	# Run unit tests
 	# Fail if coverage falls below 94%
-	LAMBDA_BUILDERS_DEV=1 pytest --cov aws_lambda_builders --cov-report term-missing --cov-fail-under 94 tests/unit tests/functional
+	LAMBDA_BUILDERS_DEV=1 pytest -vv --cov aws_lambda_builders --cov-report term-missing --cov-fail-under 94 tests/unit tests/functional
 
 func-test:
 	LAMBDA_BUILDERS_DEV=1 pytest tests/functional

aws_lambda_builders/builder.py

@@ -69,6 +69,7 @@ def build(
         dependencies_dir=None,
         combine_dependencies=True,
         architecture=X86_64,
+        is_building_layer=False,
         experimental_flags=None,
     ):
         # pylint: disable-msg=too-many-locals
@@ -130,6 +131,10 @@ def build(
         :param architecture:
             Type of architecture x86_64 and arm64 for Lambda Function
 
+        :type is_building_layer: bool
+        :param is_building_layer:
+            Boolean flag which will be set True if current build operation is being executed for layers
+
         :type experimental_flags: list
         :param experimental_flags:
             List of strings, which will indicate enabled experimental flags for the current build session
@@ -152,6 +157,7 @@ def build(
             dependencies_dir=dependencies_dir,
             combine_dependencies=combine_dependencies,
             architecture=architecture,
+            is_building_layer=is_building_layer,
             experimental_flags=experimental_flags,
         )
 

aws_lambda_builders/utils.py

@@ -12,7 +12,7 @@
 LOG = logging.getLogger(__name__)
 
 
-def copytree(source, destination, ignore=None):
+def copytree(source, destination, ignore=None, include=None):
     """
     Similar to shutil.copytree except that it removes the limitation that the destination directory should
     be present.
@@ -29,17 +29,25 @@ def copytree(source, destination, ignore=None):
     :param ignore:
         A function that returns a set of file names to ignore, given a list of available file names. Similar to the
         ``ignore`` property of ``shutils.copytree`` method
+
+    :type include: Callable[[str], bool]
+    :param include:
+        A function that will decide whether a file should be copied or skipped it. It accepts file name as parameter
+        and return True or False. Returning True will continue copy operation, returning False will skip copy operation
+        for that file
     """
 
     if not os.path.exists(source):
         LOG.warning("Skipping copy operation since source %s does not exist", source)
         return
 
     if not os.path.exists(destination):
+        LOG.debug("Creating target folders at %s", destination)
         os.makedirs(destination)
 
         try:
             # Let's try to copy the directory metadata from source to destination
+            LOG.debug("Copying directory metadata from source (%s) to destination (%s)", source, destination)
             shutil.copystat(source, destination)
         except OSError as ex:
             # Can't copy file access times in Windows
@@ -54,14 +62,20 @@ def copytree(source, destination, ignore=None):
     for name in names:
         # Skip ignored names
         if name in ignored_names:
+            LOG.debug("File (%s) is in ignored set, skipping it", name)
             continue
 
         new_source = os.path.join(source, name)
         new_destination = os.path.join(destination, name)
 
+        if include and not os.path.isdir(new_source) and not include(name):
+            LOG.debug("File (%s) doesn't satisfy the include rule, skipping it", name)
+            continue
+
         if os.path.isdir(new_source):
-            copytree(new_source, new_destination, ignore=ignore)
+            copytree(new_source, new_destination, ignore=ignore, include=include)
         else:
+            LOG.debug("Copying source file (%s) to destination (%s)", new_source, new_destination)
             shutil.copy2(new_source, new_destination)
 
 

aws_lambda_builders/workflow.py

@@ -164,8 +164,10 @@ def __init__(
         dependencies_dir=None,
         combine_dependencies=True,
         architecture=X86_64,
+        is_building_layer=False,
         experimental_flags=None,
     ):
+        # pylint: disable-msg=too-many-locals
         """
         Initialize the builder with given arguments. These arguments together form the "public API" that each
         build action must support at the minimum.
@@ -201,6 +203,10 @@ def __init__(
             from dependency_folder into build folder
         architecture : str, optional
             Architecture type either arm64 or x86_64 for which the build will be based on in AWS lambda, by default X86_64
+
+        is_building_layer: bool, optional
+            Boolean flag which will be set True if current build operation is being executed for layers
+
         experimental_flags: list, optional
             List of strings, which will indicate enabled experimental flags for the current build session
         """
@@ -218,6 +224,7 @@ def __init__(
         self.dependencies_dir = dependencies_dir
         self.combine_dependencies = combine_dependencies
         self.architecture = architecture
+        self.is_building_layer = is_building_layer
         self.experimental_flags = experimental_flags if experimental_flags else []
 
         # Actions are registered by the subclasses as they seem fit

aws_lambda_builders/workflows/java/utils.py

@@ -6,7 +6,10 @@
 import platform
 import shutil
 import subprocess
-from aws_lambda_builders.utils import which
+from aws_lambda_builders.utils import which, copytree
+
+
+EXPERIMENTAL_MAVEN_SCOPE_AND_LAYER_FLAG = "experimentalMavenScopeAndLayer"
 
 
 class OSUtils(object):
@@ -37,17 +40,8 @@ def exists(self, p):
     def which(self, executable, executable_search_paths=None):
         return which(executable, executable_search_paths=executable_search_paths)
 
-    def copytree(self, source, destination):
-        if not os.path.exists(destination):
-            self.makedirs(destination)
-        names = self.listdir(source)
-        for name in names:
-            new_source = os.path.join(source, name)
-            new_destination = os.path.join(destination, name)
-            if os.path.isdir(new_source):
-                self.copytree(new_source, new_destination)
-            else:
-                self.copy(new_source, new_destination)
+    def copytree(self, source, destination, ignore=None, include=None):
+        copytree(source, destination, ignore=ignore, include=include)
 
     def makedirs(self, d):
         return os.makedirs(d)
@@ -58,3 +52,21 @@ def rmtree(self, d):
     @property
     def pipe(self):
         return subprocess.PIPE
+
+
+def jar_file_filter(file_name):
+    """
+    A function that will filter .jar files for copy operation
+
+    :type file_name: str
+    :param file_name:
+        Name of the file that will be checked against if it ends with .jar or not
+    """
+    return bool(file_name) and isinstance(file_name, str) and file_name.endswith(".jar")
+
+
+def is_experimental_maven_scope_and_layers_active(experimental_flags):
+    """
+    A function which will determine if experimental maven scope and layer changes are active
+    """
+    return bool(experimental_flags) and EXPERIMENTAL_MAVEN_SCOPE_AND_LAYER_FLAG in experimental_flags

aws_lambda_builders/workflows/java_gradle/actions.py

@@ -5,6 +5,7 @@
 import os
 from aws_lambda_builders.actions import ActionFailedError, BaseAction, Purpose
 from .gradle import GradleExecutionError
+from ..java.utils import jar_file_filter
 
 
 class JavaGradleBuildAction(BaseAction):
@@ -56,7 +57,7 @@ def _build_project(self, init_script_file):
 
 
 class JavaGradleCopyArtifactsAction(BaseAction):
-    NAME = "CopyArtifacts"
+    NAME = "JavaGradleCopyArtifacts"
     DESCRIPTION = "Copying the built artifacts"
     PURPOSE = Purpose.COPY_SOURCE
 
@@ -77,3 +78,26 @@ def _copy_artifacts(self):
             self.os_utils.copytree(lambda_build_output, self.artifacts_dir)
         except Exception as ex:
             raise ActionFailedError(str(ex))
+
+
+class JavaGradleCopyLayerArtifactsAction(JavaGradleCopyArtifactsAction):
+    """
+    Java layers does not support using .class files in it.
+    This action (different from the parent one) copies contents of the layer as jar files and place it
+    into the artifact folder
+    """
+
+    NAME = "JavaGradleCopyLayerArtifacts"
+
+    def _copy_artifacts(self):
+        lambda_build_output = os.path.join(self.build_dir, "build", "libs")
+        layer_dependencies = os.path.join(self.build_dir, "build", "distributions", "lambda-build", "lib")
+        try:
+            if not self.os_utils.exists(self.artifacts_dir):
+                self.os_utils.makedirs(self.artifacts_dir)
+            self.os_utils.copytree(
+                lambda_build_output, os.path.join(self.artifacts_dir, "lib"), include=jar_file_filter
+            )
+            self.os_utils.copytree(layer_dependencies, os.path.join(self.artifacts_dir, "lib"), include=jar_file_filter)
+        except Exception as ex:
+            raise ActionFailedError(str(ex))

aws_lambda_builders/workflows/java_gradle/workflow.py

@@ -6,9 +6,9 @@
 from aws_lambda_builders.actions import CleanUpAction
 from aws_lambda_builders.workflow import BaseWorkflow, Capability
 from aws_lambda_builders.workflows.java.actions import JavaCopyDependenciesAction, JavaMoveDependenciesAction
-from aws_lambda_builders.workflows.java.utils import OSUtils
+from aws_lambda_builders.workflows.java.utils import OSUtils, is_experimental_maven_scope_and_layers_active
 
-from .actions import JavaGradleBuildAction, JavaGradleCopyArtifactsAction
+from .actions import JavaGradleBuildAction, JavaGradleCopyArtifactsAction, JavaGradleCopyLayerArtifactsAction
 from .gradle import SubprocessGradle
 from .gradle_resolver import GradleResolver
 from .gradle_validator import GradleValidator
@@ -33,9 +33,16 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, **kwar
 
         subprocess_gradle = SubprocessGradle(gradle_binary=self.binaries["gradle"], os_utils=self.os_utils)
 
+        copy_artifacts_action = JavaGradleCopyArtifactsAction(
+            source_dir, artifacts_dir, self.build_output_dir, self.os_utils
+        )
+        if self.is_building_layer and is_experimental_maven_scope_and_layers_active(self.experimental_flags):
+            copy_artifacts_action = JavaGradleCopyLayerArtifactsAction(
+                source_dir, artifacts_dir, self.build_output_dir, self.os_utils
+            )
         self.actions = [
             JavaGradleBuildAction(source_dir, manifest_path, subprocess_gradle, scratch_dir, self.os_utils),
-            JavaGradleCopyArtifactsAction(source_dir, artifacts_dir, self.build_output_dir, self.os_utils),
+            copy_artifacts_action,
         ]
 
         if self.dependencies_dir:

aws_lambda_builders/workflows/java_maven/DESIGN.md

@@ -79,12 +79,28 @@ source directory.
 
 ```bash
 mvn clean install
-mvn dependency:copy-dependencies -DincludeScope=compile
+mvn dependency:copy-dependencies -DincludeScope=runtime
 ```
 
+Building artifact for an `AWS::Serverless::LayerVersion` requires different packaging than a 
+`AWS::Serverless::Function`. [Layers](https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html)
+ use only artifacts under `java/lib/` which differs from Functions in that they in addition allow classes at 
+the root level similar to normal jar packaging.  `JavaMavenLayersWorkflow` handles packaging for Layers and
+`JavaMavenWorkflow` handles packaging for Functions.
+
 #### Step 4: Copy to artifact directory
 
-Built Java classes and dependencies are copied from `scratch_dir/target/classes` and `scratch_dir/target/dependency`
-to `artifact_dir` and `artifact_dir/lib` respectively.
+Built Java classes and dependencies for Functions are copied from `scratch_dir/target/classes` and `scratch_dir/target/dependency`
+to `artifact_dir` and `artifact_dir/lib` respectively. Built Java classes and dependencies for Layers are copied from 
+`scratch_dir/target/*.jar` and `scratch_dir/target/dependency` to  `artifact_dir/lib`. Copy all the artifacts 
+required for runtime execution.
+
+### Notes on changes of original implementation
+
+The original implementation was not handling Layers well.  Maven has provided a scope called `provided` which is
+used to declare that a particular dependency is required for compilation but should not be packaged with the
+declaring project artifact.  Naturally this is the scope a maven java project would use for artifacts
+provided by Layers.  Original implementation would package those `provided` scoped entities with the Function,
+and thus if a project was using Layers it would have the artifact both in the Layer and in the Function.
 
 [Gradle Lambda Builder]:https://github.com/awslabs/aws-lambda-builders/blob/develop/aws_lambda_builders/workflows/java_gradle/DESIGN.md

aws_lambda_builders/workflows/java_maven/actions.py

@@ -4,9 +4,11 @@
 
 import os
 import logging
+import shutil
 
 from aws_lambda_builders.actions import ActionFailedError, BaseAction, Purpose
 from .maven import MavenExecutionError
+from ..java.utils import jar_file_filter
 
 LOG = logging.getLogger(__name__)
 
@@ -81,3 +83,33 @@ def _copy_artifacts(self):
                 self.os_utils.copytree(dependency_output, os.path.join(self.artifacts_dir, "lib"))
         except Exception as ex:
             raise ActionFailedError(str(ex))
+
+
+class JavaMavenCopyLayerArtifactsAction(JavaMavenCopyArtifactsAction):
+    """
+    Java layers does not support using .class files in it.
+    This action (different from the parent one) copies contents of the layer as jar files and place it
+    into the artifact folder
+    """
+
+    NAME = "MavenCopyLayerArtifacts"
+    IGNORED_FOLDERS = ["classes", "dependency", "generated-sources", "maven-archiver", "maven-status"]
+
+    def _copy_artifacts(self):
+        lambda_build_output = os.path.join(self.scratch_dir, "target")
+        dependency_output = os.path.join(self.scratch_dir, "target", "dependency")
+
+        if not self.os_utils.exists(lambda_build_output):
+            raise ActionFailedError("Required target/classes directory was not produced from 'mvn package'")
+
+        try:
+            self.os_utils.copytree(
+                lambda_build_output,
+                os.path.join(self.artifacts_dir, "lib"),
+                ignore=shutil.ignore_patterns(*self.IGNORED_FOLDERS),
+                include=jar_file_filter,
+            )
+            if self.os_utils.exists(dependency_output):
+                self.os_utils.copytree(dependency_output, os.path.join(self.artifacts_dir, "lib"))
+        except Exception as ex:
+            raise ActionFailedError(str(ex))

aws_lambda_builders/workflows/java_maven/maven.py

@@ -16,13 +16,14 @@ def __init__(self, **kwargs):
 
 
 class SubprocessMaven(object):
-    def __init__(self, maven_binary, os_utils=None):
+    def __init__(self, maven_binary, os_utils=None, is_experimental_maven_scope_enabled=False):
         if maven_binary is None:
             raise ValueError("Must provide Maven BinaryPath")
         self.maven_binary = maven_binary
         if os_utils is None:
             raise ValueError("Must provide OSUtils")
         self.os_utils = os_utils
+        self.is_experimental_maven_scope_enabled = is_experimental_maven_scope_enabled
 
     def build(self, scratch_dir):
         args = ["clean", "install"]
@@ -34,7 +35,9 @@ def build(self, scratch_dir):
             raise MavenExecutionError(message=stdout.decode("utf8").strip())
 
     def copy_dependency(self, scratch_dir):
-        args = ["dependency:copy-dependencies", "-DincludeScope=compile", "-Dmdep.prependGroupId=true"]
+        include_scope = "runtime" if self.is_experimental_maven_scope_enabled else "compile"
+        LOG.debug("Running copy_dependency with scope: %s", include_scope)
+        args = ["dependency:copy-dependencies", f"-DincludeScope={include_scope}", "-Dmdep.prependGroupId=true"]
         ret_code, stdout, _ = self._run(args, scratch_dir)
 
         if ret_code != 0: