Expanded parameter overrides

samcli/__init__.py

@@ -2,4 +2,4 @@
 SAM CLI version
 """
 
-__version__ = "1.132.0"
+__version__ = "1.133.0"

samcli/cli/types.py

@@ -6,10 +6,14 @@
 import logging
 import re
 from json import JSONDecodeError
+from pathlib import Path
 from typing import Dict, List, Optional, Union
 
 import click
 
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
+
+from samcli.lib.config.file_manager import FILE_MANAGER_MAPPER
 from samcli.lib.package.ecr_utils import is_ecr_url
 
 PARAM_AND_METADATA_KEY_REGEX = """([A-Za-z0-9\\"\']+)"""
@@ -60,6 +64,28 @@ def _unquote_wrapped_quotes(value):
 
     return value.replace("\\ ", " ").replace('\\"', '"').replace("\\'", "'")
 
+def _flatten_list(data: list) -> list:
+    """
+        Recursively flattens lists and other list-like types for easy sequential processing.
+        This also helps with lists combined with YAML anchors & aliases.
+
+    Parameters
+    ----------
+    value : data
+        List to flatten
+
+    Returns
+    -------
+    Flattened list
+    """
+    flat_data = []
+    for item in data:
+        if isinstance(item, (tuple, list, CommentedSeq)):
+            flat_data.extend(_flatten_list(item))
+        else:
+            flat_data.append(item)
+    return flat_data
+
 
 class CfnParameterOverridesType(click.ParamType):
     """
@@ -69,6 +95,8 @@ class CfnParameterOverridesType(click.ParamType):
 
     __EXAMPLE_1 = "ParameterKey=KeyPairName,ParameterValue=MyKey ParameterKey=InstanceType,ParameterValue=t1.micro"
     __EXAMPLE_2 = "KeyPairName=MyKey InstanceType=t1.micro"
+    __EXAMPLE_3 = "file://MyParams.yaml"
+
 
     # Regex that parses CloudFormation parameter key-value pairs:
     # https://regex101.com/r/xqfSjW/2
@@ -86,45 +114,76 @@ class CfnParameterOverridesType(click.ParamType):
 
     ordered_pattern_match = [_pattern_1, _pattern_2]
 
-    name = "string,list"
-
-    def convert(self, value, param, ctx):
-        result = {}
-
-        # Empty tuple
-        if value == ("",):
-            return result
-
-        value = (value,) if isinstance(value, str) else value
-        for val in value:
-            # Add empty string to start of the string to help match `_pattern2`
-            normalized_val = " " + val.strip()
-
-            try:
-                # NOTE(TheSriram): find the first regex that matched.
-                # pylint is concerned that we are checking at the same `val` within the loop,
-                # but that is the point, so disabling it.
-                pattern = next(
-                    i
-                    for i in filter(
-                        lambda item: re.findall(item, normalized_val), self.ordered_pattern_match
-                    )  # pylint: disable=cell-var-from-loop
-                )
-            except StopIteration:
-                return self.fail(
-                    "{} is not in valid format. It must look something like '{}' or '{}'".format(
-                        val, self.__EXAMPLE_1, self.__EXAMPLE_2
-                    ),
+    def _normalize_parameters(self, values, param, ctx):
+        """
+            Normalizes parameter overrides into key-value pairs of strings
+            Later keys overwrite previous ones in case of key conflict
+        """
+        if values in (("",), "", None) or values == {}:
+            LOG.debug("Empty parameter set (%s)", values)
+            return {}
+
+        LOG.debug("Input parameters: %s", values)
+        values = _flatten_list([values])
+        LOG.debug("Flattened parameters: %s", values)
+
+        parameters = {}
+        for value in values:
+            if isinstance(value, str):
+                if value.startswith('file://'):
+                    filepath = Path(value[7:])
+                    if not filepath.is_file():
+                        self.fail(f"{value} was not found or is a directory", param, ctx)
+                    file_manager = FILE_MANAGER_MAPPER.get(filepath.suffix, None)
+                    if not file_manager:
+                        self.fail(f"{value} uses an unsupported extension", param, ctx)
+                    parameters |= self._normalize_parameters(file_manager.read(filepath), param, ctx)
+                else:
+                    # Legacy parameter matching
+                    normalized_value = " " + value.strip()
+                    for pattern in self.ordered_pattern_match:
+                        groups = re.findall(pattern, normalized_value)
+                        if groups:
+                            parameters |= groups
+                            break
+                    else:
+                        self.fail(
+                            f"{value} is not a valid format. It must look something like '{self.__EXAMPLE_1}', '{self.__EXAMPLE_2}', or '{self.__EXAMPLE_3}'",
+                            param,
+                            ctx,
+                        )
+            elif isinstance(value, (dict, CommentedMap)):
+                # e.g. YAML key-value pairs
+                for k, v in value.items():
+                    if isinstance(v, (list, CommentedSeq)):
+                        # Collapse list values to comma delimited
+                        parameters[str(k)] = ",".join(map(str, v))
+                    else:
+                        parameters[str(k)] = str(v)
+            else:
+                self.fail(
+                    f"{value} is not valid in a way the code doesn't expect",
                     param,
                     ctx,
                 )
 
-            groups = re.findall(pattern, normalized_val)
+        result = {}
+        for key, param_value in parameters.items():
+            result[_unquote_wrapped_quotes(key)] = _unquote_wrapped_quotes(param_value)
+        LOG.debug("Output parameters: %s", result)
+        return result
 
-            # 'groups' variable is a list of tuples ex: [(key1, value1), (key2, value2)]
-            for key, param_value in groups:
-                result[_unquote_wrapped_quotes(key)] = _unquote_wrapped_quotes(param_value)
 
+    def convert(self, values, param, ctx):
+        # Merge samconfig with CLI parameter-overrides
+        if isinstance(values, tuple): # Tuple implies CLI
+            # default_map was populated from samconfig
+            default_map = ctx.default_map.get('parameter_overrides', {})
+            LOG.debug("Default map: %s", default_map)
+            LOG.debug("Current values: %s", values)
+            # Easy merge - will flatten later
+            values = [default_map] + [values]
+        result = self._normalize_parameters(values, param, ctx)
         return result
 
 

samcli/lib/build/app_builder.py

@@ -61,6 +61,7 @@
     AWS_SERVERLESS_LAYERVERSION,
 )
 from samcli.lib.utils.stream_writer import StreamWriter
+from samcli.local.docker.container import ContainerContext
 from samcli.local.docker.lambda_build_container import LambdaBuildContainer
 from samcli.local.docker.manager import ContainerManager, DockerImagePullFailedException
 from samcli.local.docker.utils import get_docker_platform, is_docker_reachable
@@ -968,7 +969,7 @@ def _build_function_on_container(
 
         try:
             try:
-                self._container_manager.run(container)
+                self._container_manager.run(container, context=ContainerContext.BUILD)
             except docker.errors.APIError as ex:
                 if "executable file not found in $PATH" in str(ex):
                     raise UnsupportedBuilderLibraryVersionError(

samcli/local/docker/container.py

@@ -13,6 +13,7 @@
 import tempfile
 import threading
 import time
+from enum import Enum
 from typing import Dict, Iterator, Optional, Tuple, Union
 
 import docker
@@ -59,6 +60,11 @@ class ContainerConnectionTimeoutException(Exception):
     """
 
 
+class ContainerContext(Enum):
+    BUILD = "build"
+    INVOKE = "invoke"
+
+
 class Container:
     """
     Represents an instance of a Docker container with a specific configuration. The container is not actually created
@@ -157,11 +163,14 @@ def __init__(
         except NoFreePortsError as ex:
             raise ContainerNotStartableException(str(ex)) from ex
 
-    def create(self):
+    def create(self, context):
         """
         Calls Docker API to creates the Docker container instance. Creating the container does *not* run the container.
         Use ``start`` method to run the container
 
+        context: samcli.local.docker.container.ContainerContext
+            Context for the container management to run (build, invoke)
+
         :return string: ID of the created container
         :raise RuntimeError: If this method is called after a container already has been created
         """
@@ -174,6 +183,7 @@ def create(self):
         if self._host_dir:
             mount_mode = "rw,delegated" if self._mount_with_write else "ro,delegated"
             LOG.info("Mounting %s as %s:%s, inside runtime container", self._host_dir, self._working_dir, mount_mode)
+            mapped_symlinks = self._create_mapped_symlink_files() if self._resolve_symlinks(context) else {}
 
             _volumes = {
                 self._host_dir: {
@@ -182,7 +192,7 @@ def create(self):
                     "bind": self._working_dir,
                     "mode": mount_mode,
                 },
-                **self._create_mapped_symlink_files(),
+                **mapped_symlinks,
             }
 
         kwargs = {
@@ -648,3 +658,18 @@ def is_running(self):
             return real_container.status == "running"
         except docker.errors.NotFound:
             return False
+
+    def _resolve_symlinks(self, context) -> bool:
+        """_summary_
+
+        Parameters
+        ----------
+        context : sacli.local.docker.container.ContainerContext
+            Context for the container management to run. (build, invoke)
+
+        Returns
+        -------
+        bool
+            True, if given these parameters it should resolve symlinks or not
+        """
+        return bool(context != ContainerContext.BUILD)

samcli/local/docker/manager.py

@@ -12,7 +12,7 @@
 from samcli.lib.constants import DOCKER_MIN_API_VERSION
 from samcli.lib.utils.stream_writer import StreamWriter
 from samcli.local.docker import utils
-from samcli.local.docker.container import Container
+from samcli.local.docker.container import Container, ContainerContext
 from samcli.local.docker.lambda_image import LambdaImage
 
 LOG = logging.getLogger(__name__)
@@ -55,14 +55,16 @@ def is_docker_reachable(self):
         """
         return utils.is_docker_reachable(self.docker_client)
 
-    def create(self, container):
+    def create(self, container, context):
         """
         Create a container based on the given configuration.
 
         Parameters
         ----------
         container samcli.local.docker.container.Container:
             Container to be created
+        context: samcli.local.docker.container.ContainerContext
+            Context for the container management to run. (build, invoke)
 
         Raises
         ------
@@ -93,9 +95,9 @@ def create(self, container):
                 LOG.info("Failed to download a new %s image. Invoking with the already downloaded image.", image_name)
 
         container.network_id = self.docker_network_id
-        container.create()
+        container.create(context)
 
-    def run(self, container, input_data=None):
+    def run(self, container, context: ContainerContext, input_data=None):
         """
         Run a Docker container based on the given configuration.
         If the container is not created, it will call Create method to create.
@@ -104,6 +106,8 @@ def run(self, container, input_data=None):
         ----------
         container: samcli.local.docker.container.Container
             Container to create and run
+        context: samcli.local.docker.container.ContainerContext
+            Context for the container management to run. (build, invoke)
         input_data: str, optional
             Input data sent to the container through container's stdin.
 
@@ -113,8 +117,7 @@ def run(self, container, input_data=None):
             If the Docker image was not available in the server
         """
         if not container.is_created():
-            self.create(container)
-
+            self.create(container, context)
         container.start(input_data=input_data)
 
     def stop(self, container: Container) -> None:

samcli/local/lambdafn/runtime.py

@@ -14,7 +14,7 @@
 from samcli.lib.telemetry.metric import capture_parameter
 from samcli.lib.utils.file_observer import LambdaFunctionObserver
 from samcli.lib.utils.packagetype import ZIP
-from samcli.local.docker.container import Container
+from samcli.local.docker.container import Container, ContainerContext
 from samcli.local.docker.container_analyzer import ContainerAnalyzer
 from samcli.local.docker.exceptions import ContainerFailureError, DockerContainerCreationFailedException
 from samcli.local.docker.lambda_container import LambdaContainer
@@ -113,7 +113,7 @@ def create(
         )
         try:
             # create the container.
-            self._container_manager.create(container)
+            self._container_manager.create(container, ContainerContext.INVOKE)
             return container
 
         except DockerContainerCreationFailedException:
@@ -174,7 +174,7 @@ def run(
 
         try:
             # start the container.
-            self._container_manager.run(container)
+            self._container_manager.run(container, ContainerContext.INVOKE)
             return container
 
         except KeyboardInterrupt:

tests/integration/deploy/test_deploy_command.py

@@ -26,13 +26,13 @@ class TestDeploy(DeployIntegBase):
     def setUpClass(cls):
         cls.docker_client = docker.from_env()
         cls.local_images = [
-            ("public.ecr.aws/sam/emulation-python3.8", "latest"),
+            ("public.ecr.aws/sam/emulation-python3.9", "latest"),
         ]
         # setup some images locally by pulling them.
         for repo, tag in cls.local_images:
             cls.docker_client.api.pull(repository=repo, tag=tag)
-            cls.docker_client.api.tag(f"{repo}:{tag}", "emulation-python3.8", tag="latest")
-            cls.docker_client.api.tag(f"{repo}:{tag}", "emulation-python3.8-2", tag="latest")
+            cls.docker_client.api.tag(f"{repo}:{tag}", "emulation-python3.9", tag="latest")
+            cls.docker_client.api.tag(f"{repo}:{tag}", "emulation-python3.9-2", tag="latest")
             cls.docker_client.api.tag(f"{repo}:{tag}", "colorsrandomfunctionf61b9209", tag="latest")
 
         # setup signing profile arn & name

tests/integration/local/invoke/test_invoke_terraform_applications.py

@@ -210,7 +210,7 @@ def setUpClass(cls):
             bytes('resource "aws_lambda_function" "this" {' + os.linesep, "utf-8"),
             bytes("   filename = var.source_code" + os.linesep, "utf-8"),
             bytes('   handler = "app.lambda_handler"' + os.linesep, "utf-8"),
-            bytes('   runtime = "python3.8"' + os.linesep, "utf-8"),
+            bytes('   runtime = "python3.9"' + os.linesep, "utf-8"),
             bytes("   function_name = var.function_name" + os.linesep, "utf-8"),
             bytes("   role = aws_iam_role.iam_for_lambda.arn" + os.linesep, "utf-8"),
             bytes(f'   layers = ["{_4th_layer_arn}"]' + os.linesep, "utf-8"),