Allow to ignore stages executions

Author: JohnPreston

.cicd/buildspec.yml

@@ -98,6 +98,11 @@ phases:
               --amend ${PUBLIC_REGISTRY}/ecs-files-composer:${VERSION}-x86_64     \
               --amend ${PUBLIC_REGISTRY}/ecs-files-composer:${VERSION}-aarch64;
             docker manifest push ${PUBLIC_REGISTRY}/ecs-files-composer:latest;
+          else
+            docker manifest create ${PUBLIC_REGISTRY}/ecs-files-composer:nightly  \
+              --amend ${PUBLIC_REGISTRY}/ecs-files-composer:${VERSION}-x86_64     \
+              --amend ${PUBLIC_REGISTRY}/ecs-files-composer:${VERSION}-aarch64;
+            docker manifest push ${PUBLIC_REGISTRY}/ecs-files-composer:nightly;
           fi
 
     finally:

docker-compose.override.yaml

@@ -24,6 +24,7 @@ services:
       ECS_CONTAINER_METADATA_URI: http://169.254.170.2/v3/containers/files-sidecar
       AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-eu-west-1}
       context: plain
+      LOGLEVEL: debug
       ECS_CONFIG_CONTENT: |
 
         files:
@@ -34,6 +35,7 @@ services:
             group: root
             mode: 600
           /opt/files/aws.template:
+            ignore_failure: true
             source:
               S3:
                 BucketName: ${BUCKET_NAME:-sacrificial-lamb}

docker-compose.yaml

@@ -13,9 +13,8 @@ services:
     image: public.ecr.aws/compose-x/ecs-files-composer:latest
     deploy:
       resources:
-        # Smallest RAM size for a lambda
         limits:
-          cpus: 0.1
+          cpus: "0.1"
           memory: 128M
     build:
       context: ./

ecs-files-input.json

@@ -96,13 +96,43 @@
           ],
           "default": "plain"
         },
-        "ignore_if_failed": {
-          "type": "boolean",
-          "description": "Whether or not the failure to retrieve the file should stop the execution",
-          "default": false
+        "ignore_failure": {
+          "oneOf": [
+            {
+              "type": "object",
+              "additionalProperties": false,
+              "properties": {
+                "commands": {
+                  "type": "boolean",
+                  "default": false,
+                  "description": "Ignore if any of the commands failed"
+                },
+                "mode": {
+                  "type": "boolean",
+                  "default": false,
+                  "description": "Ignore if `mode` (using chmod) failed."
+                },
+                "owner": {
+                  "type": "boolean",
+                  "default": false,
+                  "description": "Ignore if `owner` (using chown) failed"
+                },
+                "source_download": {
+                  "type": "boolean",
+                  "default": false,
+                  "description": "Ignore if a Source download failed. Any subsequent action is cancelled."
+                }
+              }
+            },
+            {
+              "type": "boolean",
+              "description": "Ignore if any step fails (download, transform, commands etc.)"
+            }
+          ]
         },
         "commands": {
           "type": "object",
+          "additionalProperties": false,
           "properties": {
             "post": {
               "$ref": "#/definitions/CommandsDef",

ecs_files_composer/ecs_files_composer.py

@@ -5,22 +5,21 @@
 
 from __future__ import annotations
 
+import os
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from .input import Model
 
 import json
-import uuid
 from os import environ, path
 from tempfile import TemporaryDirectory
 
 import yaml
-from compose_x_common.compose_x_common import keyisset
 from yaml import Loader
 
 from ecs_files_composer import input
-from ecs_files_composer.aws_mgmt import S3Fetcher, SecretFetcher, SsmFetcher
+from ecs_files_composer.aws_mgmt import S3Fetcher
 from ecs_files_composer.certbot_aws_store import handle_certbot_store_certificates
 from ecs_files_composer.certificates_mgmt import process_x509_certs
 from ecs_files_composer.common import LOG
@@ -110,18 +109,22 @@ def init_config(
     if context:
         initial_config["context"] = context
     start_jobs(jobs_input_def)
-    with open(config_path) as config_fd:
-        try:
-            config = yaml.load(config_fd.read(), Loader=Loader)
-            LOG.info(f"Successfully loaded YAML config {config_path}")
-            return config
-        except yaml.YAMLError:
-            config = json.loads(config_fd.read())
-            LOG.info(f"Successfully loaded JSON config {config_path}")
-            return config
-        except Exception:
-            LOG.error("Input content is neither JSON nor YAML formatted")
-            raise
+    try:
+        with open(config_path) as config_fd:
+            try:
+                config = yaml.load(config_fd.read(), Loader=Loader)
+                LOG.info(f"Successfully loaded YAML config {config_path}")
+                return config
+            except yaml.YAMLError:
+                config = json.loads(config_fd.read())
+                LOG.info(f"Successfully loaded JSON config {config_path}")
+                return config
+            except Exception:
+                LOG.error("Input content is neither JSON nor YAML formatted")
+                raise
+    except OSError as error:
+        LOG.exception(error)
+        LOG.error(f"Failed to read input file from {config_path}")
 
 
 def process_files(job: Model, override_session=None) -> None: