Feature/http download (#1)

* Added some debug logging

* Added URL support with basic auth

* Removed troubleshooting prints

Author: JohnPreston

ecs-files-input.json

@@ -91,7 +91,7 @@
             "type": "object",
             "properties": {
                 "Url": {
-                    "type": "string"
+                    "$ref": "#/definitions/UrlDef"
                 },
                 "Ssm": {
                     "$ref": "#/definitions/SsmDef"
@@ -104,6 +104,21 @@
                 }
             }
         },
+        "UrlDef": {
+            "type": "object",
+            "properties": {
+                "Url": {
+                    "type": "string",
+                    "format": "uri"
+                },
+                "Username": {
+                    "type": "string"
+                },
+                "Password": {
+                    "type": "string"
+                }
+            }
+        },
         "SsmDef": {
             "type": "object",
             "properties": {

ecs_files_composer/cli.py

@@ -5,10 +5,9 @@
 """Console script for ecs_files_composer."""
 import argparse
 import sys
-from os import environ, path
-
-import yaml
+from os import environ
 
+from ecs_files_composer.common import LOG
 from ecs_files_composer.ecs_files_composer import init_config, start_jobs
 
 
@@ -37,8 +36,8 @@ def main():
     if not (args.env_var or args.ssm_config or args.s3_config or args.file_path) and environ.get(
         "ECS_CONFIG_CONTENT", None
     ):
+        LOG.info("Using default env variable ECS_CONFIG_CONTENT")
         config = init_config(env_var="ECS_CONFIG_CONTENT")
-        print("Config from default ECS_CONFIG_CONTENT", config)
     elif args.env_var:
         config = init_config(env_var=args.env_var)
     elif args.file_path:

ecs_files_composer/ecs_files_composer.py

@@ -13,6 +13,7 @@
 from typing import Any
 
 import boto3
+import requests
 import yaml
 from boto3 import session
 from botocore.exceptions import ClientError
@@ -146,7 +147,9 @@ def get_content(self, secret):
         :param input.SecretDef secret:
         :return:
         """
-        params = {"SecretId": expandvars(secret.secret_id)}
+        secret_id = expandvars(secret.secret_id)
+        params = {"SecretId": secret_id}
+        LOG.debug(f"Retrieving secretsmanager://{secret_id}")
         if secret.version_id:
             params["VersionId"] = secret.version_id
         if secret.version_stage:
@@ -194,7 +197,7 @@ def handle_sources(self, iam_override=None):
         :return:
         """
         if self.source.url:
-            pass
+            self.handle_http_content()
         elif self.source.ssm:
             self.handle_ssm_source(iam_override)
         elif self.source.s3:
@@ -217,6 +220,7 @@ def handle_ssm_source(self, iam_override=None):
         :return:
         """
         parameter_name = expandvars(self.source.ssm.parameter_name)
+        LOG.debug(f"Retrieving ssm://{parameter_name}")
         if self.source.ssm.iam_override:
             fetcher = SsmFetcher(iam_config_object=self.source.ssm.iam_override)
         else:
@@ -232,6 +236,7 @@ def handle_s3_source(self, iam_override=None):
         """
         bucket_name = expandvars(self.source.s3.bucket_name)
         key = expandvars(self.source.s3.key)
+        LOG.debug(f"Retrieving s3://{bucket_name}/{key}")
         if self.source.s3.iam_override:
             fetcher = S3Fetcher(iam_config_object=self.source.s3.iam_override)
         else:
@@ -251,10 +256,26 @@ def handle_secret_source(self, iam_override=None):
             fetcher = SecretFetcher(iam_config_object=iam_override)
         self.content = fetcher.get_content(self.source.secret)
 
+    def handle_http_content(self):
+        """
+        Fetches the content from a provided URI
+
+        """
+        if not self.source.url.username or not self.source.url.password:
+            req = requests.get(self.source.url.url)
+        else:
+            req = requests.get(self.source.url.url, auth=(self.source.url.username, self.source.url.password))
+        try:
+            req.raise_for_status()
+            self.write_content(as_bytes=True, bytes_content=req.content)
+        except requests.exceptions.HTTPError as e:
+            LOG.error(e)
+            raise
+
     def set_unix_settings(self):
         """
         Applies UNIX settings to given file
-        :return:
+
         """
         cmd = ["chmod", self.mode, self.path]
         try:
@@ -277,7 +298,7 @@ def set_unix_settings(self):
             else:
                 raise
 
-    def write_content(self, decode=False):
+    def write_content(self, decode=False, as_bytes=False, bytes_content=None):
         if isinstance(self.content, str):
             if decode and self.encoding == input.Encoding["base64"]:
                 with open(self.path, "wb") as file_fd:
@@ -288,6 +309,9 @@ def write_content(self, decode=False):
         elif isinstance(self.content, StreamingBody):
             with open(self.path, "wb") as file_fd:
                 file_fd.write(self.content.read())
+        elif as_bytes and bytes_content:
+            with open(self.path, "wb") as file_fd:
+                file_fd.write(bytes_content)
 
 
 def init_config(

ecs_files_composer/input.py

@@ -1,13 +1,13 @@
 # generated by datamodel-codegen:
 #   filename:  ecs-files-input.json
-#   timestamp: 2021-07-15T13:26:54+00:00
+#   timestamp: 2021-07-16T13:56:02+00:00
 
 from __future__ import annotations
 
 from enum import Enum
 from typing import Any, Dict, List, Optional
 
-from pydantic import BaseModel, Field
+from pydantic import AnyUrl, BaseModel, Field
 
 
 class Encoding(Enum):
@@ -20,6 +20,12 @@ class Context(Enum):
     jinja2 = 'jinja2'
 
 
+class UrlDef(BaseModel):
+    url: Optional[AnyUrl] = Field(None, alias='Url')
+    username: Optional[str] = Field(None, alias='Username')
+    password: Optional[str] = Field(None, alias='Password')
+
+
 class IamOverrideDef(BaseModel):
     role_arn: Optional[str] = Field(None, alias='RoleArn')
     session_name: Optional[str] = Field(
@@ -71,7 +77,7 @@ class S3Def(BaseModel):
 
 
 class SourceDef(BaseModel):
-    url: Optional[str] = Field(None, alias='Url')
+    url: Optional[UrlDef] = Field(None, alias='Url')
     ssm: Optional[SsmDef] = Field(None, alias='Ssm')
     s3: Optional[S3Def] = Field(None, alias='S3')
     secret: Optional[SecretDef] = Field(None, alias='Secret')