Drop pydantic and use dataclasses instead. (#24)

Author: JohnPreston

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
       - id: end-of-file-fixer
       - id: trailing-whitespace
@@ -17,14 +17,14 @@ repos:
       - id: fix-byte-order-marker
 
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.10.1
+    rev: v3.15.0
     hooks:
       - id: pyupgrade
         args: [ "--py38-plus" ]
         exclude: ^ecs_files_composer/input.py|docs/
 
   - repo: https://github.com/psf/black
-    rev: 23.7.0
+    rev: 23.10.0
     hooks:
       - id: black
   - repo: https://github.com/pycqa/isort

Dockerfile

@@ -1,5 +1,5 @@
 ARG ARCH=
-ARG PY_VERSION=3.9
+ARG PY_VERSION=3.10
 ARG BASE_IMAGE=public.ecr.aws/docker/library/python:${PY_VERSION}-slim
 ARG LAMBDA_IMAGE=public.ecr.aws/lambda/python:latest
 
@@ -9,14 +9,13 @@ WORKDIR /opt
 RUN yum install gcc -y
 COPY ecs_files_composer /opt/ecs_files_composer
 COPY poetry.lock pyproject.toml MANIFEST.in README.rst LICENSE /opt/
-RUN python -m pip install pip -U; python -m pip install poetry; poetry build
+RUN python -m pip install pip -U; python -m pip install poetry; poetry build; poetry export -o /opt/requirements.txt
 
 
 FROM $BASE_IMAGE
-
 COPY --from=builder /opt/dist/*.whl ${LAMBDA_TASK_ROOT:-/app/}/dist/
-
-RUN python -m pip install pip -U --no-cache-dir; \
-    python -m pip install --no-cache-dir /app/dist/*.whl
+COPY --from=builder /opt/requirements.txt ${LAMBDA_TASK_ROOT:-/app/}/requirements.txt
+RUN python -m pip install pip -U --no-cache-dir; pip install --no-cache-dir -r /app/requirements.txt
+RUN python -m pip install --no-cache-dir /app/dist/*.whl
 WORKDIR /
 ENTRYPOINT ["python", "-m", "ecs_files_composer.cli"]

docs/commands.rst

@@ -0,0 +1,70 @@
+.. meta::
+    :description: Files Commands
+    :keywords: Configuration, commands
+
+.. _commands_docs:
+
+Commands
+=========
+
+.. warning::
+
+    When using docker, the container user is root to simplify access to filesystem.
+
+Within files, you have the ability to run `post` commands, which are commands to run after writing the file to disk
+was done successfully.
+
+
+Commands can come in handy to change permissions on files and all sorts of things so that the application that will
+read/consume the file & folders will have permissions to do so.
+
+Commands can be written in 2 ways
+
+* a simple string: will execute the command, won't output the result
+* a mapping, with the command, and options
+
+
+Using the mapping option
+-------------------------
+
+As of 2.0, the available options are
+
+* display_output: bool
+* ignore_error: bool
+
+
+display_output
+^^^^^^^^^^^^^^^
+
+As its name would suggest, this will log the output of the command.
+
+.. warning::
+
+    Use carefully as it won't be filtering out any secret values & details
+
+ignore_error
+^^^^^^^^^^^^^^^
+
+You can already set error conditions and ignore at the top level of the file. This option allows for a localized setting.
+For example, if every other commands must succeed but one of them can fail without causing an error, you would set this to true
+
+In the example below, we want to see the content of `/tmp/testing/toto.txt` which we know must work because
+it will be rendered.
+
+However, file `/tmp/init/init.con` might not exist on the file system, so if it doesn't, we ignore.
+
+.. code-block:: yaml
+
+files:
+  /tmp/testing/toto.txt:
+    content: |
+      HELLO YOU
+    context: plain
+    commands:
+      post:
+        - chown 1000:1000 -R /opt/files
+        - command: ls /tmp/
+          display_output: true
+        - command: cat /tmp/init/init.con
+          display_output: true
+          ignore_error: true

docs/index.rst

@@ -88,6 +88,7 @@ docker-compose
     input
     features
     jinja2_functions_filters
+    commands
     certificates
     examples
     modules

ecs-files-input.json

@@ -284,8 +284,31 @@
       "type": "array",
       "description": "List of commands to run",
       "items": {
-        "type": "string",
-        "description": "Shell command to run"
+        "oneOf": [
+          {
+            "type": "string",
+            "description": "Shell command to run"
+          },
+          {
+            "type": "object",
+            "description": "Command to run with options",
+            "properties": {
+              "command": {
+                "type": "string"
+              },
+              "display_output": {
+                "type": "boolean",
+                "default": false,
+                "description": "Displays the command output"
+              },
+              "ignore_error": {
+                "type": "boolean",
+                "description": "Ignore if command failed",
+                "default": false
+              }
+            }
+          }
+        ]
       }
     },
     "X509CertDef": {

ecs_files_composer/aws_mgmt.py

@@ -23,12 +23,12 @@ def create_session_from_creds(tmp_creds: dict, region: str = None):
     """
     creds = tmp_creds["Credentials"]
     params = {
-        "aws_access_key_id": creds["AccessKeyId"],
-        "aws_secret_access_key": creds["SecretAccessKey"],
+        "aws_AccessKeyId": creds["AccessKeyId"],
+        "aws_SecretAccessKey": creds["SecretAccessKey"],
         "aws_session_token": creds["SessionToken"],
     }
     if region:
-        params["region_name"] = region
+        params["RegionName"] = region
     return boto3.session.Session(**params)
 
 
@@ -43,23 +43,23 @@ def set_session_from_iam_object(iam_config_object, source_session: Session = Non
     """
     if source_session is None:
         source_session = boto3.Session()
-    if not iam_config_object.access_key_id and not iam_config_object.secret_access_key:
+    if not iam_config_object.AccessKeyId and not iam_config_object.SecretAccessKey:
         params = {
-            "RoleArn": iam_config_object.role_arn,
-            "RoleSessionName": f"{iam_config_object.session_name}@AwsResourceHandlerInit",
+            "RoleArn": iam_config_object.RoleArn,
+            "RoleSessionName": f"{iam_config_object.SessionName}@AwsResourceHandlerInit",
         }
-        if iam_config_object.external_id:
-            params["ExternalId"] = iam_config_object.external_id
+        if iam_config_object.ExternalId:
+            params["ExternalId"] = iam_config_object.ExternalId
         tmp_creds = source_session.client("sts").assume_role(**params)
         client_session = create_session_from_creds(
-            tmp_creds, region=iam_config_object.region_name
+            tmp_creds, region=iam_config_object.RegionName
         )
     else:
         client_session = boto3.session.Session(
-            aws_access_key_id=iam_config_object.access_key_id,
-            aws_secret_access_key=iam_config_object.secret_access_key,
-            aws_session_token=iam_config_object.session_token
-            if iam_config_object.session_token
+            aws_access_key_id=iam_config_object.AccessKeyId,
+            aws_secret_access_key=iam_config_object.SecretAccessKey,
+            aws_session_token=iam_config_object.SessionToken
+            if iam_config_object.SessionToken
             else None,
         )
     return client_session
@@ -72,38 +72,38 @@ class AwsResourceHandler:
 
     def __init__(
         self,
-        role_arn=None,
-        external_id=None,
+        RoleArn=None,
+        ExternalId=None,
         region=None,
         iam_config_object=None,
         client_session_override=None,
     ):
         """
-        :param str role_arn:
-        :param str external_id:
+        :param str RoleArn:
+        :param str ExternalId:
         :param str region:
         :param ecs_files_composer.input.IamOverrideDef iam_config_object:
         """
         self.session = Session()
         self.client_session = Session()
         if client_session_override:
             self.client_session = client_session_override
-        elif not client_session_override and (role_arn or iam_config_object):
-            if role_arn and not iam_config_object:
+        elif not client_session_override and (RoleArn or iam_config_object):
+            if RoleArn and not iam_config_object:
                 params = {
-                    "RoleArn": role_arn,
+                    "RoleArn": RoleArn,
                     "RoleSessionName": "EcsConfigComposer@AwsResourceHandlerInit",
                 }
-                if external_id:
-                    params["ExternalId"] = external_id
+                if ExternalId:
+                    params["ExternalId"] = ExternalId
                 tmp_creds = self.session.client("sts").assume_role(**params)
                 self.client_session = create_session_from_creds(
                     tmp_creds, region=region
                 )
             elif (
                 iam_config_object
-                and hasattr(iam_config_object, "role_arn")
-                and iam_config_object.role_arn
+                and hasattr(iam_config_object, "RoleArn")
+                and iam_config_object.RoleArn
             ):
                 self.client_session = set_session_from_iam_object(
                     iam_config_object, self.session
@@ -120,14 +120,14 @@ class S3Fetcher(AwsResourceHandler):
 
     def __init__(
         self,
-        role_arn=None,
-        external_id=None,
+        RoleArn=None,
+        ExternalId=None,
         region=None,
         iam_config_object=None,
         client_session_override=None,
     ):
         super().__init__(
-            role_arn, external_id, region, iam_config_object, client_session_override
+            RoleArn, ExternalId, region, iam_config_object, client_session_override
         )
 
     @property
@@ -173,14 +173,14 @@ class SsmFetcher(AwsResourceHandler):
 
     def __init__(
         self,
-        role_arn=None,
-        external_id=None,
+        RoleArn=None,
+        ExternalId=None,
         region=None,
         iam_config_object=None,
         client_session_override=None,
     ):
         super().__init__(
-            role_arn, external_id, region, iam_config_object, client_session_override
+            RoleArn, ExternalId, region, iam_config_object, client_session_override
         )
         self.client = self.client_session.client("ssm")
 
@@ -212,14 +212,14 @@ class SecretFetcher(AwsResourceHandler):
 
     def __init__(
         self,
-        role_arn=None,
-        external_id=None,
+        RoleArn=None,
+        ExternalId=None,
         region=None,
         iam_config_object=None,
         client_session_override=None,
     ):
         super().__init__(
-            role_arn, external_id, region, iam_config_object, client_session_override
+            RoleArn, ExternalId, region, iam_config_object, client_session_override
         )
         self.client = self.client_session.client("secretsmanager")
 
@@ -230,13 +230,13 @@ def get_content(self, secret):
         :param input.SecretDef secret:
         :return:
         """
-        secret_id = expandvars(secret.secret_id)
+        secret_id = expandvars(secret.SecretId)
         params = {"SecretId": secret_id}
         LOG.debug(f"Retrieving secretsmanager://{secret_id}")
-        if secret.version_id:
-            params["VersionId"] = secret.version_id
-        if secret.version_stage:
-            params["VersionStage"] = secret.version_stage
+        if secret.VersionId:
+            params["VersionId"] = secret.VersionId
+        if secret.VersionStage:
+            params["VersionStage"] = secret.VersionStage
         try:
             parameter = self.client.get_secret_value(**params)
             return parameter["SecretString"]