From 34dd0c7dd7cf169e250a4a167a32ddbfa3517cd7 Mon Sep 17 00:00:00 2001 From: Joey Hage <3220319+joeyhage@users.noreply.github.com> Date: Fri, 1 Aug 2025 17:00:00 +0000 Subject: [PATCH] feat: add python option for creating parent packages during copy --- aws_lambda_builders/builder.py | 2 +- aws_lambda_builders/workflow.py | 2 +- .../workflows/python_pip/DESIGN.md | 44 +++++++++++++++ .../workflows/python_pip/actions.py | 49 +++++++++++++++++ .../workflows/python_pip/workflow.py | 36 ++++++++++--- .../workflows/python_pip/test_python_pip.py | 52 ++++++++++++++++-- tests/unit/test_builder.py | 6 +-- .../unit/workflows/python_pip/test_actions.py | 54 +++++++++++++++++-- 8 files changed, 224 insertions(+), 21 deletions(-) diff --git a/aws_lambda_builders/builder.py b/aws_lambda_builders/builder.py index e47c299e9..ad9d0cb37 100644 --- a/aws_lambda_builders/builder.py +++ b/aws_lambda_builders/builder.py @@ -105,7 +105,7 @@ def build( :type options: dict :param options: - Optional dictionary of options ot pass to build action. **Not supported**. + Optional dictionary of options to pass to build action. :type executable_search_paths: list :param executable_search_paths: diff --git a/aws_lambda_builders/workflow.py b/aws_lambda_builders/workflow.py index d8897882e..9e2ce835f 100644 --- a/aws_lambda_builders/workflow.py +++ b/aws_lambda_builders/workflow.py @@ -227,7 +227,7 @@ def __init__( optimizations : dict, optional dictionary of optimization flags to pass to the build action. **Not supported**, by default None options : dict, optional - dictionary of options ot pass to build action. **Not supported**., by default None + dictionary of options to pass to build action. By default None mode : str, optional Mode the build should produce, by default BuildMode.RELEASE download_dependencies: bool, optional diff --git a/aws_lambda_builders/workflows/python_pip/DESIGN.md b/aws_lambda_builders/workflows/python_pip/DESIGN.md index 58f1ef308..77ebc5abc 100644 --- a/aws_lambda_builders/workflows/python_pip/DESIGN.md +++ b/aws_lambda_builders/workflows/python_pip/DESIGN.md @@ -147,3 +147,47 @@ bundle has an `__init__.py` and is on the `PYTHONPATH`. The dependencies should now be succesfully installed in the target directory. All the temporary/intermediate files can now be deleting including all the wheel files and sdists. + +### Configuring the builder + +The Lambda builder supports the following optional sub-properties of the `aws_sam` configuration property. + +#### `parent_python_packages` + +`parent_python_packages` must be a string, corresponding to a dot-separated list of parent packages to create in the destination directory. This is useful when the source code has a package structure that needs to be preserved in the built artifacts. + +For example, if your source code is structured like this: + +``` +├── app +| ├── main.py +| └── utils +| └── logger.py +└── tests + └── unit/test_handler.py +``` + +Then the SAM build output will look like this: + +``` +└── .aws-sam + └── build + └── AppLogicalId + ├── main.py + └── utils + └── logger.py +``` + +The tests would break because `main.py` is importing `logger` as `from .utils import logger`, but the built artifacts would not have the `app` package. + +To fix this, you can set the `parent_python_packages` property to `app` and the SAM build output will look like this: + +``` +└── .aws-sam + └── build + └── AppLogicalId + └── app + ├── main.py + └── utils + └── logger.py +``` \ No newline at end of file diff --git a/aws_lambda_builders/workflows/python_pip/actions.py b/aws_lambda_builders/workflows/python_pip/actions.py index dcef47a96..81b8dae65 100644 --- a/aws_lambda_builders/workflows/python_pip/actions.py +++ b/aws_lambda_builders/workflows/python_pip/actions.py @@ -3,6 +3,7 @@ """ import logging +from pathlib import Path from typing import Optional, Tuple from aws_lambda_builders.actions import ActionFailedError, BaseAction, Purpose @@ -21,6 +22,8 @@ LOG = logging.getLogger(__name__) +PARENT_PYTHON_PKGS_KEY = "parent_python_packages" + class PythonPipBuildAction(BaseAction): NAME = "ResolveDependencies" @@ -115,3 +118,49 @@ def _find_runtime_with_pip(self) -> Tuple[SubprocessPip, str]: LOG.debug(f"Python runtime path '{python_path}' does not contain pip") raise ActionFailedError("Failed to find a Python runtime containing pip on the PATH.") + + +class PythonCreateParentPackagesAction(BaseAction): + NAME = "CreateParentPackages" + DESCRIPTION = "Creating parent Python packages" + PURPOSE = Purpose.COPY_SOURCE + LANGUAGE = "python" + + def __init__(self, source_dir, dest_dir, options=None): + self.source_dir = source_dir + self.dest_dir = dest_dir + self.options = options or {} + + def execute(self): + pkg_parts = self._get_parent_python_packages() + if not pkg_parts: + return + + source_path = Path(self.source_dir) + dest_path = Path(self.dest_dir) + + target_pkg_path = dest_path.joinpath(*pkg_parts) + try: + target_pkg_path.mkdir(parents=True, exist_ok=False) + except FileExistsError: + LOG.warning("Skipping creating package %s as it would overwrite existing folder", target_pkg_path) + return + + for item in source_path.glob(pattern="*"): + if (dest_path / item.name).exists(): + (dest_path / item.name).rename(target_pkg_path / item.name) + else: + LOG.debug(f"{item} does not exist in the build path, skipping.") + + def _get_parent_python_packages(self) -> Optional[list]: + """ + Returns the parent Python packages to be created. + """ + if not isinstance(self.options, dict): + return None + + parent_python_pkgs = self.options.get(PARENT_PYTHON_PKGS_KEY) + if isinstance(parent_python_pkgs, str) and len(parent_python_pkgs) > 0: + return parent_python_pkgs.split(".") + + return None diff --git a/aws_lambda_builders/workflows/python_pip/workflow.py b/aws_lambda_builders/workflows/python_pip/workflow.py index 1862fd450..6273277aa 100644 --- a/aws_lambda_builders/workflows/python_pip/workflow.py +++ b/aws_lambda_builders/workflows/python_pip/workflow.py @@ -9,7 +9,7 @@ from aws_lambda_builders.workflow import BaseWorkflow, BuildDirectory, BuildInSourceSupport, Capability from aws_lambda_builders.workflows.python_pip.validator import PythonRuntimeValidator -from .actions import PythonPipBuildAction +from .actions import PARENT_PYTHON_PKGS_KEY, PythonCreateParentPackagesAction, PythonPipBuildAction from .utils import OSUtils, is_experimental_build_improvements_enabled LOG = logging.getLogger(__name__) @@ -87,15 +87,15 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim self.actions = [] if not osutils.file_exists(manifest_path): LOG.warning("requirements.txt file not found. Continuing the build without dependencies.") - self.actions.append(CopySourceAction(source_dir, artifacts_dir, excludes=self.EXCLUDED_FILES)) + self._actions.append(CopySourceAction(source_dir, artifacts_dir, excludes=self.EXCLUDED_FILES)) return # If a requirements.txt exists, run pip builder before copy action. if self.download_dependencies: if self.dependencies_dir: # clean up the dependencies folder before installing - self.actions.append(CleanUpAction(self.dependencies_dir)) - self.actions.append( + self._actions.append(CleanUpAction(self.dependencies_dir)) + self._actions.append( PythonPipBuildAction( artifacts_dir, scratch_dir, @@ -113,11 +113,25 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim # when copying downloaded dependencies back to artifacts folder, don't exclude anything # symlinking python dependencies is disabled for now since it is breaking sam local commands if False and is_experimental_build_improvements_enabled(self.experimental_flags): - self.actions.append(LinkSourceAction(self.dependencies_dir, artifacts_dir)) + self._actions.append(LinkSourceAction(self.dependencies_dir, artifacts_dir)) else: - self.actions.append(CopySourceAction(self.dependencies_dir, artifacts_dir)) + self._actions.append(CopySourceAction(self.dependencies_dir, artifacts_dir)) - self.actions.append(CopySourceAction(source_dir, artifacts_dir, excludes=self.EXCLUDED_FILES)) + self._actions.append(CopySourceAction(source_dir, artifacts_dir, excludes=self.EXCLUDED_FILES)) + + @property + def actions(self): + """ + Returns the list of actions to be executed in this workflow. + If required, creating the parent package(s) must be executed last. + """ + if self._should_create_parent_packages(): + return self._actions + [PythonCreateParentPackagesAction(self.source_dir, self.artifacts_dir, self.options)] + return self._actions + + @actions.setter + def actions(self, value): + self._actions = value def get_resolvers(self): """ @@ -138,5 +152,13 @@ def _get_additional_binaries(self): major, _ = self.runtime.replace(self.CAPABILITY.language, "").split(".") return [f"{self.CAPABILITY.language}{major}"] if major == self.PYTHON_VERSION_THREE else None + def _should_create_parent_packages(self): + """ + Determines if the parent package(s) should be created based on the options provided. + """ + if isinstance(self.options, dict): + return PARENT_PYTHON_PKGS_KEY in self.options + return False + def get_validators(self): return [PythonRuntimeValidator(runtime=self.runtime, architecture=self.architecture)] diff --git a/tests/integration/workflows/python_pip/test_python_pip.py b/tests/integration/workflows/python_pip/test_python_pip.py index 162921645..297a2c21b 100644 --- a/tests/integration/workflows/python_pip/test_python_pip.py +++ b/tests/integration/workflows/python_pip/test_python_pip.py @@ -1,17 +1,16 @@ +import logging import os import pathlib +import platform import shutil import sys -import platform import tempfile -from unittest import TestCase, skipIf, mock +from unittest import TestCase, mock, skipIf from parameterized import parameterized_class from aws_lambda_builders.builder import LambdaBuilder from aws_lambda_builders.exceptions import WorkflowFailedError -import logging - from aws_lambda_builders.utils import which from aws_lambda_builders.workflows.python_pip.utils import EXPERIMENTAL_FLAG_BUILD_PERFORMANCE @@ -432,3 +431,48 @@ def test_without_combine_dependencies(self): dependencies_files = set(os.listdir(self.dependencies_dir)) for f in expected_dependencies_files: self.assertIn(f, dependencies_files) + + def test_must_copy_with_parent_packages_without_manifest(self): + parent_package = "testdata" + self.builder.build( + self.source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join("non", "existent", "manifest"), + runtime=self.runtime, + experimental_flags=self.experimental_flags, + options={"parent_python_packages": parent_package}, + ) + self.assertEqual(set([parent_package]), set(os.listdir(self.artifacts_dir))) + expected_files = self.test_data_files + output_files = set(os.listdir(pathlib.Path(self.artifacts_dir) / parent_package)) + self.assertEqual(expected_files, output_files) + + def test_must_copy_with_parent_packages_with_manifest(self): + parent_package = "testdata" + self.builder.build( + self.source_dir, + self.artifacts_dir, + self.scratch_dir, + self.manifest_path_valid, + runtime=self.runtime, + experimental_flags=self.experimental_flags, + options={"parent_python_packages": parent_package}, + ) + + if self.runtime in ("python3.12", "python3.13"): + self.check_architecture_in("numpy-2.1.2.dist-info", ["manylinux2014_x86_64", "manylinux1_x86_64"]) + expected_dependencies = {"numpy", "numpy-2.1.2.dist-info", "numpy.libs"} + elif self.runtime in ("python3.10", "python3.11"): + self.check_architecture_in("numpy-1.23.5.dist-info", ["manylinux2014_x86_64", "manylinux1_x86_64"]) + expected_dependencies = {"numpy", "numpy-1.23.5.dist-info", "numpy.libs"} + else: + self.check_architecture_in("numpy-1.20.3.dist-info", ["manylinux2010_x86_64", "manylinux1_x86_64"]) + expected_dependencies = {"numpy", "numpy-1.20.3.dist-info", "numpy.libs"} + + output_files: set[str] = set(os.listdir(self.artifacts_dir)) + self.assertEqual(set([parent_package, *expected_dependencies]), output_files) + + expected_files = self.test_data_files + output_files = set(os.listdir(pathlib.Path(self.artifacts_dir) / parent_package)) + self.assertEqual(expected_files, output_files) diff --git a/tests/unit/test_builder.py b/tests/unit/test_builder.py index 9dad6d80b..7f42a5fef 100644 --- a/tests/unit/test_builder.py +++ b/tests/unit/test_builder.py @@ -1,12 +1,12 @@ import itertools from unittest import TestCase -from unittest.mock import patch, call, Mock +from unittest.mock import Mock, call, patch from parameterized import parameterized from aws_lambda_builders.builder import LambdaBuilder -from aws_lambda_builders.workflow import BuildDirectory, BuildInSourceSupport, Capability, BaseWorkflow from aws_lambda_builders.registry import DEFAULT_REGISTRY +from aws_lambda_builders.workflow import BaseWorkflow, BuildDirectory, BuildInSourceSupport, Capability class TesetLambdaBuilder_init(TestCase): @@ -193,6 +193,6 @@ def test_with_mocks( workflow_instance.run.assert_called_once() os_mock.path.exists.assert_called_once_with("scratch_dir") if scratch_dir_exists: - os_mock.makedirs.not_called() + os_mock.makedirs.assert_not_called() else: os_mock.makedirs.assert_called_once_with("scratch_dir") diff --git a/tests/unit/workflows/python_pip/test_actions.py b/tests/unit/workflows/python_pip/test_actions.py index 8cd43c047..33bf9c9a9 100644 --- a/tests/unit/workflows/python_pip/test_actions.py +++ b/tests/unit/workflows/python_pip/test_actions.py @@ -1,15 +1,17 @@ import sys - from unittest import TestCase -from unittest.mock import patch, Mock, ANY +from unittest.mock import ANY, MagicMock, Mock, patch from aws_lambda_builders.actions import ActionFailedError from aws_lambda_builders.architecture import ARM64, X86_64 from aws_lambda_builders.binary_path import BinaryPath - -from aws_lambda_builders.workflows.python_pip.actions import PythonPipBuildAction +from aws_lambda_builders.workflows.python_pip.actions import ( + PARENT_PYTHON_PKGS_KEY, + PythonCreateParentPackagesAction, + PythonPipBuildAction, +) from aws_lambda_builders.workflows.python_pip.exceptions import MissingPipError -from aws_lambda_builders.workflows.python_pip.packager import PackagerError, SubprocessPip +from aws_lambda_builders.workflows.python_pip.packager import PackagerError class TestPythonPipBuildAction(TestCase): @@ -185,3 +187,45 @@ def test_find_runtime_no_python_matches(self): PythonPipBuildAction(Mock(), Mock(), Mock(), Mock(), Mock(), mock_binaries)._find_runtime_with_pip() self.assertEqual(str(ex.exception), "Failed to find a Python runtime containing pip on the PATH.") + + +class TestPythonCreateParentPackagesAction(TestCase): + def setUp(self): + self.source = "source" + self.dest = "dest" + + self.mock_source_dir = MagicMock(name="source_dir") + self.mock_source_file = MagicMock(name="source_file") + self.mock_dest_dir = MagicMock(name="dest_dir") + self.mock_dest_file = MagicMock(name="dest_file") + self.target_dir = MagicMock(name="target_dir") + + self.mock_dest_dir.joinpath.return_value = self.target_dir + self.mock_source_dir.glob.return_value = [self.mock_source_file] + self.mock_dest_dir.__truediv__.return_value = self.mock_dest_file + + def mock_source_dest(self, mock_path): + mock_path.side_effect = lambda x: self.mock_source_dir if x == self.source else self.mock_dest_dir + + @patch("aws_lambda_builders.workflows.python_pip.actions.Path") + def test_skips_bad_config(self, mock_path): + self.mock_source_dest(mock_path) + action = PythonCreateParentPackagesAction(self.source, self.dest, options="not_a_dict") + + action.execute() + + self.mock_dest_file.rename.assert_not_called() + + @patch("aws_lambda_builders.workflows.python_pip.actions.Path") + def test_creates_parent_packages(self, mock_path): + self.mock_source_dest(mock_path) + + self.mock_dest_file.exists.return_value = True + + action = PythonCreateParentPackagesAction( + self.source, self.dest, options={PARENT_PYTHON_PKGS_KEY: "foo.bar.baz"} + ) + + action.execute() + + self.mock_dest_file.rename.assert_called_once_with(self.target_dir / "baz")