Added examples and fixed env var import

Author: JohnPreston

Dockerfile

@@ -18,4 +18,3 @@ WORKDIR /app
 RUN echo $PATH ; pip install pip -U --no-cache-dir && pip install wheel --no-cache-dir && pip install *.whl --no-cache-dir
 WORKDIR /
 ENTRYPOINT ["ecs_files_composer"]
-CMD ["-h"]

docker-compose.override.yaml

@@ -0,0 +1,51 @@
+---
+# Override file for local testing with config from the default env var
+
+version: "3.8"
+services:
+  ecs-local-endpoints:
+    image: amazon/amazon-ecs-local-container-endpoints:latest-amd64
+    volumes:
+      - /var/run:/var/run
+      - $HOME/.aws/:/home/.aws/
+    environment:
+      ECS_LOCAL_METADATA_PORT: "51679"
+      HOME: "/home"
+      AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-eu-west-1}
+      AWS_PROFILE: ${AWS_PROFILE}
+    ports:
+      - 51679:51679
+
+  files-sidecar:
+    environment:
+      AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: "/creds"
+      AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-eu-west-1}
+      ECS_CONFIG_CONTENT: |
+
+        files:
+          /opt/files/test.txt:
+            content: >-
+              test from a yaml raw content
+            owner: john
+            group: root
+            mode: 600
+          /opt/files/aws.template:
+            source:
+              S3:
+                BucketName: ${BUCKET_NAME:-sacrificial-lamb}
+                Key: aws.yml
+
+          /opt/files/ssm.txt:
+            source:
+              Ssm:
+                ParameterName: /cicd/shared/kms/arn
+            commands:
+              post:
+                - file /opt/files/ssm.txt
+
+          /opt/files/secret.txt:
+            source:
+              Secret:
+                SecretId: GHToken
+    depends_on:
+      - ecs-local-endpoints

docker-compose.yaml

@@ -0,0 +1,21 @@
+---
+# Docker compose base
+
+version: "3.8"
+
+volumes:
+  localshared:
+
+services:
+  files-sidecar:
+    volumes:
+      - localshared:/opt/files/
+    image: public.ecr.aws/compose-x/ecs-files-composer:v0.0.1
+    deploy:
+      resources:
+        # Smallest RAM size for a lambda
+        limits:
+          cpus: 0.1
+          memory: 128M
+    build:
+      context: ./

ecs_files_composer/cli.py

@@ -37,9 +37,10 @@ def main():
     if not (args.env_var or args.ssm_config or args.s3_config or args.file_path) and environ.get(
         "ECS_CONFIG_CONTENT", None
     ):
-        config = environ.get("ECS_CONFIG_CONTENT")
+        config = init_config(env_var="ECS_CONFIG_CONTENT")
+        print("Config from default ECS_CONFIG_CONTENT", config)
     elif args.env_var:
-        config = environ.get(args.env_var)
+        config = init_config(env_var=args.env_var)
     elif args.file_path:
         config = init_config(file_path=args.file_path)
     elif args.ssm_config:

ecs_files_composer/ecs_files_composer.py

@@ -8,21 +8,19 @@
 import json
 import re
 import subprocess
-import tempfile
 import warnings
 from os import environ, path
 from typing import Any
 
 import boto3
 import yaml
-from boto3 import client, session
+from boto3 import session
 from botocore.exceptions import ClientError
 from botocore.response import StreamingBody
-from yaml import Dumper, Loader
+from yaml import Loader
 
 from ecs_files_composer import input
-from ecs_files_composer.chmod import chmod
-from ecs_files_composer.common import LOG, keyisset, keypresent
+from ecs_files_composer.common import LOG, keyisset
 from ecs_files_composer.envsubst import expandvars
 
 
@@ -328,13 +326,15 @@ def init_config(
         raise Exception("No input source was provided")
     if not config_content:
         raise ImportError("Failed to import a configuration content")
+    LOG.debug(config_content)
     try:
         config = yaml.load(config_content, Loader=Loader)
     except yaml.YAMLError:
         config = json.loads(config_content)
     except Exception:
         LOG.error("Input content is neither JSON nor YAML formatted")
         raise
+    LOG.debug(config)
     return config
 
 

examples/kafka-connect.config.yaml

@@ -0,0 +1,13 @@
+files:
+  /opt/connect/truststore.jks:
+    mode: 644
+    source:
+      S3:
+        BucketName: ${CONNECT_BUCKET}
+        Key: /cluster/${ENV}/truststore.jks
+  /opt/connect/keystore.jks:
+    mode: 644
+    source:
+      S3:
+        BucketName: ${CONNECT_BUCKET}
+        Key: /cluster/${ENV}/keystore.jks

examples/kafka-connect.yaml

@@ -0,0 +1,72 @@
+---
+# Example for injecting files into Kafka connect
+# Here we need to retrieve JKS files for our connect clients to connect to Kafka cluster(s)
+
+version: "3.8"
+
+services:
+
+  files-sidecar:
+    deploy:
+      labels:
+        ecs.task.family: connect
+        ecs.depends.condition: SUCCESS
+    volumes:
+    - localshared:/opt/connect
+    environment:
+      ENV: dev
+      ECS_CONFIG_CONTENT: |
+
+        files:
+          /opt/connect/truststore.jks:
+            mode: 644
+            source:
+              S3:
+                BucketName: ${CONNECT_BUCKET}
+                Key: /cluster/${ENV}/truststore.jks
+          /opt/connect/keystore.jks:
+            mode: 644
+            source:
+              S3:
+                BucketName: ${CONNECT_BUCKET}
+                Key: /cluster/${ENV}/keystore.jks
+
+
+  # Here we have a very simple definition for the connect service. Note that we mount the same docker volume.
+  # To avoid confusion we mount it to the same mount point as the files-sidecar, but that could change to another one.
+  # With the deploy labels, we indicate to ECS Compose-X to group these two services into the same task definition.
+  # The depends_on helps us to define that the files sidecar need to be started first
+
+  kafka-connect:
+    image: public.ecr.aws/ews-network/confluentinc/cp-kafka-connect:6.2.0
+    volumes:
+    - localshared:/opt/connect/
+
+    deploy:
+      resources:
+        reservations:
+          cpus: "2.0"
+          memory: 2GB
+        limits:
+          cpus: "2.0"
+          memory: "3.5GB"
+      labels:
+        ecs.task.family: connect
+    depends_on:
+      - files-sidecar
+
+# The following section works with ECS Compose-X which will retrieve information about the bucket in the account,
+# provide IAM access to the TaskRole and expose an environment variable CONNECT_BUCKET to the files-sidecar.
+
+x-s3:
+  connect-bucket:
+    Lookup:
+      Tags:
+        - BucketName: connect-bucket
+    Settings:
+      EnvNames: CONNECT_BUCKET
+    Services:
+      - name: files-sidecar
+        access:
+          bucket: ListOnly
+          objects: ReadOnly