Support S3Uri and ComposeXUri paths to S3

Author: JohnPreston

ecs-files-input.json

@@ -41,6 +41,16 @@
     }
   },
   "definitions": {
+    "S3Uri": {
+      "type": "string",
+      "description": "s3://bucket-name/path/to/file simple syntax. Does not support IamOverride",
+      "pattern": "^s3://([a-zA-Z\\d\\-.]+)/([\\S]+)$"
+    },
+    "ComposeXUri": {
+      "type": "string",
+      "description": "bucket_name::path/to/file format used in other compose-x projects",
+      "pattern": "([a-zA-Z\\-\\d.]+)::([\\S]+)$"
+    },
     "FileDef": {
       "type": "object",
       "additionalProperties": true,
@@ -175,11 +185,31 @@
     },
     "S3Def": {
       "type": "object",
-      "required": [
-        "BucketName",
-        "Key"
+      "oneOf": [
+        {
+          "required": [
+            "BucketName",
+            "Key"
+          ]
+        },
+        {
+          "required": [
+            "S3Uri"
+          ]
+        },
+        {
+          "required": [
+            "ComposeXUri"
+          ]
+        }
       ],
       "properties": {
+        "S3Uri": {
+          "$ref": "#/definitions/S3Uri"
+        },
+        "ComposeXUri": {
+          "$ref": "#/definitions/ComposeXUri"
+        },
         "BucketName": {
           "type": "string",
           "description": "Name of the S3 Bucket"

ecs_files_composer/aws_mgmt.py

@@ -115,7 +115,8 @@ class S3Fetcher(AwsResourceHandler):
     Class to handle S3 actions
     """
 
-    bucket_re = re.compile(r"(?:s3://)(?P<bucket>[a-z0-9-.]+)/(?P<key>[\S]+$)")
+    bucket_re = re.compile(r"^s3://(?P<bucket>[a-zA-Z\d\-.]+)/(?P<key>[\S]+)$")
+    compose_x_re = re.compile(r"^(?P<bucket>[a-zA-Z\d\-.]+)::(?P<key>[\S]+)$")
 
     def __init__(
         self,
@@ -128,21 +129,30 @@ def __init__(
         super().__init__(
             role_arn, external_id, region, iam_config_object, client_session_override
         )
-        self.client = self.client_session.client("s3")
 
-    def get_content(self, s3_uri=None, s3_bucket=None, s3_key=None):
+    @property
+    def client(self):
+        return self.client_session.client("s3")
+
+    def get_content(
+        self,
+        s3_uri: str = None,
+        s3_bucket: str = None,
+        s3_key: str = None,
+        composex_uri: str = None,
+    ):
         """
         Retrieves a file in a temp dir and returns content
 
-        :param str s3_uri:
-        :param str s3_bucket:
-        :param str s3_key:
         :return: The Stream Body for the file, allowing to do various things
         """
 
-        if s3_uri and self.bucket_re.match(s3_uri).groups():
+        if s3_uri and self.bucket_re.match(s3_uri):
             s3_bucket = self.bucket_re.match(s3_uri).group("bucket")
             s3_key = self.bucket_re.match(s3_uri).group("key")
+        elif composex_uri and self.compose_x_re.match(composex_uri):
+            s3_bucket = self.compose_x_re.match(composex_uri).group("bucket")
+            s3_key = self.compose_x_re.match(composex_uri).group("key")
         try:
             file_r = self.client.get_object(Bucket=s3_bucket, Key=s3_key)
             file_content = file_r["Body"]
@@ -158,7 +168,7 @@ class SsmFetcher(AwsResourceHandler):
     """
 
     arn_re = re.compile(
-        r"(?:^arn:aws(?:-[a-z]+)?:ssm:[\S]+:[0-9]+:parameter)(?P<name>/[\S]+)$"
+        r"^arn:aws(?:-[a-z]+)?:ssm:[\S]+:[\d]{12}:parameter(?P<name>/[\S]+)$"
     )
 
     def __init__(

ecs_files_composer/files_mgmt.py

@@ -118,18 +118,33 @@ def handle_s3_source(self, iam_override=None, session_override=None):
         :param boto3.session.Session session_override:
         :return:
         """
-        bucket_name = expandvars(self.source.s3.bucket_name)
-        key = expandvars(self.source.s3.key)
-        LOG.debug(f"Retrieving s3://{bucket_name}/{key}")
-        if self.source.s3.iam_override:
-            fetcher = S3Fetcher(iam_config_object=self.source.s3.iam_override)
+        from ecs_files_composer.input import S3Def1
+
+        if not isinstance(self.source.s3.__root__, S3Def1):
+            raise TypeError(
+                "S3 source is not of type S3Def1", type(self.source.s3.__root__)
+            )
+
+        if self.source.s3.__root__.iam_override:
+            fetcher = S3Fetcher(iam_config_object=self.source.s3.__root__.iam_override)
         elif iam_override:
             fetcher = S3Fetcher(iam_config_object=iam_override)
         elif session_override:
             fetcher = S3Fetcher(client_session_override=session_override)
         else:
             fetcher = S3Fetcher()
-        self.content = fetcher.get_content(s3_bucket=bucket_name, s3_key=key)
+        if self.source.s3.__root__.s3_uri:
+            self.content = fetcher.get_content(
+                s3_uri=self.source.s3.__root__.s3_uri.__root__
+            )
+        elif self.source.s3.__root__.compose_x_uri:
+            self.content = fetcher.get_content(
+                composex_uri=self.source.s3.__root__.compose_x_uri.__root__
+            )
+        else:
+            bucket_name = expandvars(self.source.s3.__root__.bucket_name)
+            key = expandvars(self.source.s3.__root__.key)
+            self.content = fetcher.get_content(s3_bucket=bucket_name, s3_key=key)
 
     def handle_secret_source(self, iam_override=None, session_override=None):
         """

ecs_files_composer/input.py

@@ -1,15 +1,31 @@
 # generated by datamodel-codegen:
 #   filename:  ecs-files-input.json
-#   timestamp: 2022-07-28T10:48:50+00:00
+#   timestamp: 2022-08-07T22:11:46+00:00
 
 from __future__ import annotations
 
 from enum import Enum
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional, Union
 
 from pydantic import AnyUrl, BaseModel, EmailStr, Extra, Field, constr
 
 
+class S3Uri(BaseModel):
+    __root__: str = Field(
+        ...,
+        description="s3://bucket-name/path/to/file simple syntax. Does not support IamOverride",
+        regex="^s3://([a-zA-Z\\d\\-.]+)/([\\S]+)$",
+    )
+
+
+class ComposeXUri(BaseModel):
+    __root__: str = Field(
+        ...,
+        description="bucket_name::path/to/file format used in other compose-x projects",
+        regex="([a-zA-Z\\-\\d.]+)::([\\S]+)$",
+    )
+
+
 class Encoding(Enum):
     base64 = "base64"
     plain = "plain"
@@ -136,19 +152,27 @@ class SecretDef(BaseModel):
     iam_override: Optional[IamOverrideDef] = Field(None, alias="IamOverride")
 
 
-class S3Def(BaseModel):
-    bucket_name: str = Field(
-        ..., alias="BucketName", description="Name of the S3 Bucket"
+class S3Def1(BaseModel):
+    s3_uri: Optional[S3Uri] = Field(None, alias="S3Uri")
+    compose_x_uri: Optional[ComposeXUri] = Field(None, alias="ComposeXUri")
+    bucket_name: Optional[str] = Field(
+        None, alias="BucketName", description="Name of the S3 Bucket"
     )
     bucket_region: Optional[str] = Field(
         None,
         alias="BucketRegion",
         description="S3 Region to use. Default will ignore or retrieve via s3:GetBucketLocation",
     )
-    key: str = Field(..., alias="Key", description="Full path to the file to retrieve")
+    key: Optional[str] = Field(
+        None, alias="Key", description="Full path to the file to retrieve"
+    )
     iam_override: Optional[IamOverrideDef] = Field(None, alias="IamOverride")
 
 
+class S3Def(BaseModel):
+    __root__: Union[S3Def1, Any, Any, Any]
+
+
 class SourceDef(BaseModel):
     url: Optional[UrlDef] = Field(None, alias="Url")
     ssm: Optional[SsmDef] = Field(None, alias="Ssm")

pyproject.toml

@@ -21,13 +21,13 @@ keywords=["aws", "ecs", "k8s", "secrets", "certbot"]
 
 [tool.poetry.dependencies]
 python = "^3.8"
-boto3 = "^1.23.2"
+boto3 = "^1.24"
 pydantic = {extras = ["email"], version = "^1.9.0"}
 pyOpenSSL = "^22"
 requests = "^2.27.1"
 PyYAML = "^6.0"
 Jinja2 = "^3.1.2"
-jsonschema = "^4.5.1"
+jsonschema = "^4"
 compose-x-common = "^1.0"
 certbot-aws-store = "^0.2.2"