feat: add python option for creating parent packages during copy

Author: joeyhage

aws_lambda_builders/builder.py

@@ -105,7 +105,7 @@ def build(
 
         :type options: dict
         :param options:
-            Optional dictionary of options ot pass to build action. **Not supported**.
+            Optional dictionary of options to pass to build action.
 
         :type executable_search_paths: list
         :param executable_search_paths:

aws_lambda_builders/workflow.py

@@ -227,7 +227,7 @@ def __init__(
         optimizations : dict, optional
             dictionary of optimization flags to pass to the build action. **Not supported**, by default None
         options : dict, optional
-            dictionary of options ot pass to build action. **Not supported**., by default None
+            dictionary of options to pass to build action. By default None
         mode : str, optional
             Mode the build should produce, by default BuildMode.RELEASE
         download_dependencies: bool, optional

aws_lambda_builders/workflows/python_pip/DESIGN.md

@@ -147,3 +147,47 @@ bundle has an `__init__.py` and is on the `PYTHONPATH`.
 The dependencies should now be succesfully installed in the target directory.
 All the temporary/intermediate files can now be deleting including all the
 wheel files and sdists.
+
+### Configuring the builder
+
+The Lambda builder supports the following optional sub-properties of the `aws_sam` configuration property.
+
+#### `parent_python_packages`
+
+`parent_python_packages` must be a string, corresponding to a dot-separated list of parent packages to create in the destination directory. This is useful when the source code has a package structure that needs to be preserved in the built artifacts.
+
+For example, if your source code is structured like this:
+
+```
+├── app
+|   ├── main.py
+|   └── utils
+|       └── logger.py
+└── tests
+    └── unit/test_handler.py
+```
+
+Then the SAM build output will look like this:
+
+```
+└── .aws-sam
+    └── build
+       └── AppLogicalId
+          ├── main.py
+          └── utils
+                └── logger.py
+```
+
+The tests would break because `main.py` is importing `logger` as `from .utils import logger`, but the built artifacts would not have the `app` package.
+
+To fix this, you can set the `parent_python_packages` property to `app` and the SAM build output will look like this:
+
+```
+└── .aws-sam
+    └── build
+       └── AppLogicalId
+          └── app
+              ├── main.py
+              └── utils
+                    └── logger.py
+```

aws_lambda_builders/workflows/python_pip/actions.py

@@ -3,6 +3,7 @@
 """
 
 import logging
+from pathlib import Path
 from typing import Optional, Tuple
 
 from aws_lambda_builders.actions import ActionFailedError, BaseAction, Purpose
@@ -21,6 +22,8 @@
 
 LOG = logging.getLogger(__name__)
 
+PARENT_PYTHON_PKGS_KEY = "parent_python_packages"
+
 
 class PythonPipBuildAction(BaseAction):
     NAME = "ResolveDependencies"
@@ -115,3 +118,49 @@ def _find_runtime_with_pip(self) -> Tuple[SubprocessPip, str]:
                 LOG.debug(f"Python runtime path '{python_path}' does not contain pip")
 
         raise ActionFailedError("Failed to find a Python runtime containing pip on the PATH.")
+
+
+class PythonCreateParentPackagesAction(BaseAction):
+    NAME = "CreateParentPackages"
+    DESCRIPTION = "Creating parent Python packages"
+    PURPOSE = Purpose.COPY_SOURCE
+    LANGUAGE = "python"
+
+    def __init__(self, source_dir, dest_dir, options=None):
+        self.source_dir = source_dir
+        self.dest_dir = dest_dir
+        self.options = options or {}
+
+    def execute(self):
+        pkg_parts = self._get_parent_python_packages()
+        if not pkg_parts:
+            return
+
+        source_path = Path(self.source_dir)
+        dest_path = Path(self.dest_dir)
+
+        target_pkg_path = dest_path.joinpath(*pkg_parts)
+        try:
+            target_pkg_path.mkdir(parents=True, exist_ok=False)
+        except FileExistsError:
+            LOG.warning("Skipping creating package %s as it would overwrite existing folder", target_pkg_path)
+            return
+
+        for item in source_path.glob(pattern="*"):
+            if (dest_path / item.name).exists():
+                (dest_path / item.name).rename(target_pkg_path / item.name)
+            else:
+                LOG.debug(f"{item} does not exist in the build path, skipping.")
+
+    def _get_parent_python_packages(self) -> Optional[list]:
+        """
+        Returns the parent Python packages to be created.
+        """
+        if not isinstance(self.options, dict):
+            return None
+
+        parent_python_pkgs = self.options.get(PARENT_PYTHON_PKGS_KEY)
+        if isinstance(parent_python_pkgs, str) and len(parent_python_pkgs) > 0:
+            return parent_python_pkgs.split(".")
+
+        return None

aws_lambda_builders/workflows/python_pip/workflow.py

@@ -9,7 +9,7 @@
 from aws_lambda_builders.workflow import BaseWorkflow, BuildDirectory, BuildInSourceSupport, Capability
 from aws_lambda_builders.workflows.python_pip.validator import PythonRuntimeValidator
 
-from .actions import PythonPipBuildAction
+from .actions import PARENT_PYTHON_PKGS_KEY, PythonCreateParentPackagesAction, PythonPipBuildAction
 from .utils import OSUtils, is_experimental_build_improvements_enabled
 
 LOG = logging.getLogger(__name__)
@@ -87,15 +87,15 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim
         self.actions = []
         if not osutils.file_exists(manifest_path):
             LOG.warning("requirements.txt file not found. Continuing the build without dependencies.")
-            self.actions.append(CopySourceAction(source_dir, artifacts_dir, excludes=self.EXCLUDED_FILES))
+            self._actions.append(CopySourceAction(source_dir, artifacts_dir, excludes=self.EXCLUDED_FILES))
             return
 
         # If a requirements.txt exists, run pip builder before copy action.
         if self.download_dependencies:
             if self.dependencies_dir:
                 # clean up the dependencies folder before installing
-                self.actions.append(CleanUpAction(self.dependencies_dir))
-            self.actions.append(
+                self._actions.append(CleanUpAction(self.dependencies_dir))
+            self._actions.append(
                 PythonPipBuildAction(
                     artifacts_dir,
                     scratch_dir,
@@ -113,11 +113,25 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim
             # when copying downloaded dependencies back to artifacts folder, don't exclude anything
             # symlinking python dependencies is disabled for now since it is breaking sam local commands
             if False and is_experimental_build_improvements_enabled(self.experimental_flags):
-                self.actions.append(LinkSourceAction(self.dependencies_dir, artifacts_dir))
+                self._actions.append(LinkSourceAction(self.dependencies_dir, artifacts_dir))
             else:
-                self.actions.append(CopySourceAction(self.dependencies_dir, artifacts_dir))
+                self._actions.append(CopySourceAction(self.dependencies_dir, artifacts_dir))
 
-        self.actions.append(CopySourceAction(source_dir, artifacts_dir, excludes=self.EXCLUDED_FILES))
+        self._actions.append(CopySourceAction(source_dir, artifacts_dir, excludes=self.EXCLUDED_FILES))
+
+    @property
+    def actions(self):
+        """
+        Returns the list of actions to be executed in this workflow.
+        If required, creating the parent package(s) must be executed last.
+        """
+        if self._should_create_parent_packages():
+            return self._actions + [PythonCreateParentPackagesAction(self.source_dir, self.artifacts_dir, self.options)]
+        return self._actions
+
+    @actions.setter
+    def actions(self, value):
+        self._actions = value
 
     def get_resolvers(self):
         """
@@ -138,5 +152,13 @@ def _get_additional_binaries(self):
         major, _ = self.runtime.replace(self.CAPABILITY.language, "").split(".")
         return [f"{self.CAPABILITY.language}{major}"] if major == self.PYTHON_VERSION_THREE else None
 
+    def _should_create_parent_packages(self):
+        """
+        Determines if the parent package(s) should be created based on the options provided.
+        """
+        if isinstance(self.options, dict):
+            return PARENT_PYTHON_PKGS_KEY in self.options
+        return False
+
     def get_validators(self):
         return [PythonRuntimeValidator(runtime=self.runtime, architecture=self.architecture)]

tests/integration/workflows/python_pip/test_python_pip.py

@@ -1,17 +1,16 @@
+import logging
 import os
 import pathlib
+import platform
 import shutil
 import sys
-import platform
 import tempfile
-from unittest import TestCase, skipIf, mock
+from unittest import TestCase, mock, skipIf
 
 from parameterized import parameterized_class
 
 from aws_lambda_builders.builder import LambdaBuilder
 from aws_lambda_builders.exceptions import WorkflowFailedError
-import logging
-
 from aws_lambda_builders.utils import which
 from aws_lambda_builders.workflows.python_pip.utils import EXPERIMENTAL_FLAG_BUILD_PERFORMANCE
 
@@ -432,3 +431,48 @@ def test_without_combine_dependencies(self):
         dependencies_files = set(os.listdir(self.dependencies_dir))
         for f in expected_dependencies_files:
             self.assertIn(f, dependencies_files)
+
+    def test_must_copy_with_parent_packages_without_manifest(self):
+        parent_package = "testdata"
+        self.builder.build(
+            self.source_dir,
+            self.artifacts_dir,
+            self.scratch_dir,
+            os.path.join("non", "existent", "manifest"),
+            runtime=self.runtime,
+            experimental_flags=self.experimental_flags,
+            options={"parent_python_packages": parent_package},
+        )
+        self.assertEqual(set([parent_package]), set(os.listdir(self.artifacts_dir)))
+        expected_files = self.test_data_files
+        output_files = set(os.listdir(pathlib.Path(self.artifacts_dir) / parent_package))
+        self.assertEqual(expected_files, output_files)
+
+    def test_must_copy_with_parent_packages_with_manifest(self):
+        parent_package = "testdata"
+        self.builder.build(
+            self.source_dir,
+            self.artifacts_dir,
+            self.scratch_dir,
+            self.manifest_path_valid,
+            runtime=self.runtime,
+            experimental_flags=self.experimental_flags,
+            options={"parent_python_packages": parent_package},
+        )
+
+        if self.runtime in ("python3.12", "python3.13"):
+            self.check_architecture_in("numpy-2.1.2.dist-info", ["manylinux2014_x86_64", "manylinux1_x86_64"])
+            expected_dependencies = {"numpy", "numpy-2.1.2.dist-info", "numpy.libs"}
+        elif self.runtime in ("python3.10", "python3.11"):
+            self.check_architecture_in("numpy-1.23.5.dist-info", ["manylinux2014_x86_64", "manylinux1_x86_64"])
+            expected_dependencies = {"numpy", "numpy-1.23.5.dist-info", "numpy.libs"}
+        else:
+            self.check_architecture_in("numpy-1.20.3.dist-info", ["manylinux2010_x86_64", "manylinux1_x86_64"])
+            expected_dependencies = {"numpy", "numpy-1.20.3.dist-info", "numpy.libs"}
+
+        output_files: set[str] = set(os.listdir(self.artifacts_dir))
+        self.assertEqual(set([parent_package, *expected_dependencies]), output_files)
+
+        expected_files = self.test_data_files
+        output_files = set(os.listdir(pathlib.Path(self.artifacts_dir) / parent_package))
+        self.assertEqual(expected_files, output_files)

tests/unit/test_builder.py

@@ -1,12 +1,12 @@
 import itertools
 from unittest import TestCase
-from unittest.mock import patch, call, Mock
+from unittest.mock import Mock, call, patch
 
 from parameterized import parameterized
 
 from aws_lambda_builders.builder import LambdaBuilder
-from aws_lambda_builders.workflow import BuildDirectory, BuildInSourceSupport, Capability, BaseWorkflow
 from aws_lambda_builders.registry import DEFAULT_REGISTRY
+from aws_lambda_builders.workflow import BaseWorkflow, BuildDirectory, BuildInSourceSupport, Capability
 
 
 class TesetLambdaBuilder_init(TestCase):
@@ -193,6 +193,6 @@ def test_with_mocks(
         workflow_instance.run.assert_called_once()
         os_mock.path.exists.assert_called_once_with("scratch_dir")
         if scratch_dir_exists:
-            os_mock.makedirs.not_called()
+            os_mock.makedirs.assert_not_called()
         else:
             os_mock.makedirs.assert_called_once_with("scratch_dir")

tests/unit/workflows/python_pip/test_actions.py

@@ -1,15 +1,17 @@
 import sys
-
 from unittest import TestCase
-from unittest.mock import patch, Mock, ANY
+from unittest.mock import ANY, MagicMock, Mock, patch
 
 from aws_lambda_builders.actions import ActionFailedError
 from aws_lambda_builders.architecture import ARM64, X86_64
 from aws_lambda_builders.binary_path import BinaryPath
-
-from aws_lambda_builders.workflows.python_pip.actions import PythonPipBuildAction
+from aws_lambda_builders.workflows.python_pip.actions import (
+    PARENT_PYTHON_PKGS_KEY,
+    PythonCreateParentPackagesAction,
+    PythonPipBuildAction,
+)
 from aws_lambda_builders.workflows.python_pip.exceptions import MissingPipError
-from aws_lambda_builders.workflows.python_pip.packager import PackagerError, SubprocessPip
+from aws_lambda_builders.workflows.python_pip.packager import PackagerError
 
 
 class TestPythonPipBuildAction(TestCase):
@@ -185,3 +187,45 @@ def test_find_runtime_no_python_matches(self):
             PythonPipBuildAction(Mock(), Mock(), Mock(), Mock(), Mock(), mock_binaries)._find_runtime_with_pip()
 
             self.assertEqual(str(ex.exception), "Failed to find a Python runtime containing pip on the PATH.")
+
+
+class TestPythonCreateParentPackagesAction(TestCase):
+    def setUp(self):
+        self.source = "source"
+        self.dest = "dest"
+
+        self.mock_source_dir = MagicMock(name="source_dir")
+        self.mock_source_file = MagicMock(name="source_file")
+        self.mock_dest_dir = MagicMock(name="dest_dir")
+        self.mock_dest_file = MagicMock(name="dest_file")
+        self.target_dir = MagicMock(name="target_dir")
+
+        self.mock_dest_dir.joinpath.return_value = self.target_dir
+        self.mock_source_dir.glob.return_value = [self.mock_source_file]
+        self.mock_dest_dir.__truediv__.return_value = self.mock_dest_file
+
+    def mock_source_dest(self, mock_path):
+        mock_path.side_effect = lambda x: self.mock_source_dir if x == self.source else self.mock_dest_dir
+
+    @patch("aws_lambda_builders.workflows.python_pip.actions.Path")
+    def test_skips_bad_config(self, mock_path):
+        self.mock_source_dest(mock_path)
+        action = PythonCreateParentPackagesAction(self.source, self.dest, options="not_a_dict")
+
+        action.execute()
+
+        self.mock_dest_file.rename.assert_not_called()
+
+    @patch("aws_lambda_builders.workflows.python_pip.actions.Path")
+    def test_creates_parent_packages(self, mock_path):
+        self.mock_source_dest(mock_path)
+
+        self.mock_dest_file.exists.return_value = True
+
+        action = PythonCreateParentPackagesAction(
+            self.source, self.dest, options={PARENT_PYTHON_PKGS_KEY: "foo.bar.baz"}
+        )
+
+        action.execute()
+
+        self.mock_dest_file.rename.assert_called_once_with(self.target_dir / "baz")