Skip to content

Commit 9c24a69

Browse files
JohnPrestonJohnPreston
committed
ECS specific improvements
* On init, context should not be Jinja by default * Added some ECS specific functions to retrieve metadata information
1 parent 78e8863 commit 9c24a69

File tree

6 files changed

+278
-9
lines changed

6 files changed

+278
-9
lines changed

‎docker-compose.override.yaml‎

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,16 @@ services:
1212
ECS_LOCAL_METADATA_PORT: "51679"
1313
HOME: "/home"
1414
AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-eu-west-1}
15-
AWS_PROFILE: ${AWS_PROFILE}
15+
AWS_PROFILE: ${AWS_PROFILE:-default}
1616
ports:
1717
- 51679:51679
18+
container_name: ecs-local-endpoints
1819

1920
files-sidecar:
21+
container_name: files-sidecar
2022
environment:
2123
AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: "/creds"
24+
ECS_CONTAINER_METADATA_URI: http://169.254.170.2/v3/containers/files-sidecar
2225
AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-eu-west-1}
2326
ECS_CONFIG_CONTENT: |
2427
@@ -47,7 +50,30 @@ services:
4750
source:
4851
Secret:
4952
SecretId: GHToken
50-
# command: --from-s3 s3://sacrificial-lamb/test.yaml
53+
54+
/opt/files/test_ecs_property.txt:
55+
content: |
56+
{{ ecs_container_metadata('PrivateDNSName') }}
57+
context: jinja2
58+
59+
/opt/files/test_ecs_properties.yaml:
60+
content: |
61+
{{ ecs_container_metadata() | to_yaml }}
62+
context: jinja2
63+
/opt/files/test_ecs_properties.json:
64+
content: |
65+
{{ ecs_task_metadata() | tojson }}
66+
# HELLO
67+
{{ ecs_container_metadata('ImageID')}}
68+
context: jinja2
5169
5270
depends_on:
5371
- ecs-local-endpoints
72+
73+
volumes:
74+
localshared:
75+
driver: local
76+
driver_opts:
77+
device: /tmp/files
78+
o: bind
79+
type: none

‎docker-compose.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ services:
1010
files-sidecar:
1111
volumes:
1212
- localshared:/opt/files/
13-
image: public.ecr.aws/compose-x/ecs-files-composer:v0.1.1
13+
image: public.ecr.aws/compose-x/ecs-files-composer:latest
1414
deploy:
1515
resources:
1616
# Smallest RAM size for a lambda

‎ecs_files_composer/ecs_files_composer.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ def init_config(
9393
}
9494
if decode_base64:
9595
initial_config["encoding"] = "base64"
96-
initial_config["context"] = "jinja2"
96+
# initial_config["context"] = "jinja2"
9797
start_jobs(jobs_input_def)
9898
with open(config_path, "r") as config_fd:
9999
try:

‎ecs_files_composer/files_mgmt.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
from ecs_files_composer.common import LOG
2222
from ecs_files_composer.envsubst import expandvars
2323
from ecs_files_composer.input import Context, Encoding, FileDef
24-
from ecs_files_composer.jinja2_filters import env as env_override
24+
from ecs_files_composer.jinja2_filters import JINJA_FILTERS, JINJA_FUNCTIONS
2525

2626

2727
class File(FileDef, object):
@@ -177,13 +177,13 @@ def render_jinja(self):
177177
a final template.
178178
"""
179179
LOG.info(f"Rendering Jinja for {self.path} - {self.templates_dir.name}")
180-
print(self.content)
181180
jinja_env = Environment(
182181
loader=FileSystemLoader(self.templates_dir.name),
183182
autoescape=True,
184183
auto_reload=False,
185184
)
186-
jinja_env.filters["env_override"] = env_override
185+
jinja_env.filters.update(JINJA_FILTERS)
186+
jinja_env.globals.update(JINJA_FUNCTIONS)
187187
template = jinja_env.get_template(path.basename(self.path))
188188
self.content = template.render(env=os.environ)
189189
self.write_content(is_template=False)

‎ecs_files_composer/jinja2_filters/__init__.py‎

Lines changed: 127 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,140 @@
55
"""
66
Package allowing to expand the Jinja filters to use.
77
"""
8-
8+
import json
9+
import re
910
from os import environ
1011

12+
import requests
13+
import yaml
14+
1115

12-
def env(value, key):
16+
def env_override(value, key):
1317
"""
1418
Function to use in new Jinja filter
1519
:param value:
1620
:param key:
1721
:return:
1822
"""
1923
return environ.get(key, value)
24+
25+
26+
def define_metadata(for_task=False):
27+
meta_v4 = "ECS_CONTAINER_METADATA_URI_V4"
28+
meta_v3 = "ECS_CONTAINER_METADATA_URI"
29+
30+
if environ.get(meta_v3, None):
31+
meta_url = environ.get(meta_v3)
32+
elif environ.get(meta_v4, None):
33+
meta_url = environ.get(meta_v4)
34+
else:
35+
raise EnvironmentError(
36+
"No ECS Metadata URL provided. This filter only works on ECS"
37+
)
38+
if for_task:
39+
return requests.get(f"{meta_url}/task")
40+
else:
41+
return requests.get(meta_url)
42+
43+
44+
def from_list_to_dict(top_key, new_mapping, to_convert):
45+
"""
46+
47+
:param str top_key:
48+
:param dict new_mapping:
49+
:param list to_convert:
50+
:return:
51+
"""
52+
for count, item in enumerate(to_convert):
53+
list_key = f"{top_key}_{count}"
54+
if isinstance(item, dict):
55+
from_dict_to_simple_keys(list_key, new_mapping, item)
56+
elif isinstance(item, list):
57+
from_list_to_dict(list_key, new_mapping, item)
58+
59+
60+
def from_dict_to_simple_keys(top_key, new_mapping, to_convert):
61+
"""
62+
63+
:param str top_key:
64+
:param dict new_mapping:
65+
:param dict to_convert:
66+
:return:
67+
"""
68+
for key, value in to_convert.items():
69+
if isinstance(value, str):
70+
new_mapping[f"{top_key}_{key}"] = value
71+
elif isinstance(value, dict):
72+
from_dict_to_simple_keys(f"{top_key}_{key}", new_mapping, value)
73+
elif isinstance(value, list):
74+
from_list_to_dict(f"{top_key}_{key}", new_mapping, value)
75+
76+
77+
def from_metadata_to_flat_keys(metadata):
78+
"""
79+
Function to transform the metadata into simplified structure
80+
:param dict metadata:
81+
:return:
82+
"""
83+
new_metadata = {}
84+
for key, value in metadata.items():
85+
if isinstance(value, str):
86+
new_metadata[key] = value
87+
elif isinstance(value, dict):
88+
from_dict_to_simple_keys(key, new_metadata, value)
89+
elif isinstance(value, list):
90+
from_list_to_dict(key, new_metadata, value)
91+
return new_metadata
92+
93+
94+
def get_property(metadata, property_key):
95+
metadata_mapping = from_metadata_to_flat_keys(metadata)
96+
property_re = re.compile(property_key)
97+
for key, value in metadata_mapping.items():
98+
if property_re.findall(key):
99+
return value
100+
return None
101+
102+
103+
def ecs_container_metadata(property_key=None, fallback_value=None):
104+
metadata_raw = define_metadata()
105+
metadata = metadata_raw.json()
106+
if property_key:
107+
value = get_property(metadata, property_key)
108+
if value is None:
109+
print(f"No task property found matching {property_key}")
110+
return fallback_value
111+
return value
112+
return metadata
113+
114+
115+
def ecs_task_metadata(property_key=None, fallback_value=None):
116+
metadata_raw = define_metadata(for_task=True)
117+
metadata = metadata_raw.json()
118+
if property_key:
119+
value = get_property(metadata, property_key)
120+
if value is None:
121+
print(f"No task property found matching {property_key}")
122+
return fallback_value
123+
return value
124+
return metadata
125+
126+
127+
def to_yaml(value):
128+
"""
129+
Filter to render input to YAML formatted content
130+
:return:
131+
"""
132+
return yaml.dump(value, Dumper=yaml.Dumper)
133+
134+
135+
def to_json(value, indent=2):
136+
return json.dumps(value, indent=indent)
137+
138+
139+
JINJA_FUNCTIONS = {
140+
"ecs_container_metadata": ecs_container_metadata,
141+
"ecs_task_metadata": ecs_task_metadata,
142+
}
143+
144+
JINJA_FILTERS = {"to_yaml": to_yaml, "to_json": to_json, "env_override": env_override}

‎tests/test_ecs_metadata.py‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
#!/usr/bin/env python
2+
# -*- coding: utf-8 -*-
3+
# SPDX-License-Identifier: MPL-2.0
4+
# Copyright 2020-2021 John Mille<[email protected]>
5+
6+
import json
7+
import uuid
8+
from base64 import b64encode
9+
from os import path
10+
11+
import boto3.session
12+
import pytest
13+
14+
from ecs_files_composer import input
15+
from ecs_files_composer.ecs_files_composer import start_jobs
16+
17+
HERE = path.abspath(path.dirname(__file__))
18+
19+
test_container = {
20+
"DockerId": "cd189a933e5849daa93386466019ab50-2495160603",
21+
"Name": "curl",
22+
"DockerName": "curl",
23+
"Image": "111122223333.dkr.ecr.us-west-2.amazonaws.com/curltest:latest",
24+
"ImageID": "sha256:25f3695bedfb454a50f12d127839a68ad3caf91e451c1da073db34c542c4d2cb",
25+
"Labels": {
26+
"com.amazonaws.ecs.cluster": "arn:aws:ecs:us-west-2:111122223333:cluster/default",
27+
"com.amazonaws.ecs.container-name": "curl",
28+
"com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/cd189a933e5849daa93386466019ab50",
29+
"com.amazonaws.ecs.task-definition-family": "curltest",
30+
"com.amazonaws.ecs.task-definition-version": "2",
31+
},
32+
"DesiredStatus": "RUNNING",
33+
"KnownStatus": "RUNNING",
34+
"Limits": {"CPU": 10, "Memory": 128},
35+
"CreatedAt": "2020-10-08T20:09:11.44527186Z",
36+
"StartedAt": "2020-10-08T20:09:11.44527186Z",
37+
"Type": "NORMAL",
38+
"Networks": [
39+
{
40+
"NetworkMode": "awsvpc",
41+
"IPv4Addresses": ["192.0.2.3"],
42+
"AttachmentIndex": 0,
43+
"MACAddress": "0a:de:f6:10:51:e5",
44+
"IPv4SubnetCIDRBlock": "192.0.2.0/24",
45+
"DomainNameServers": ["192.0.2.2"],
46+
"DomainNameSearchList": ["us-west-2.compute.internal"],
47+
"PrivateDNSName": "ip-10-0-0-222.us-west-2.compute.internal",
48+
"SubnetGatewayIpv4Address": "192.0.2.0/24",
49+
}
50+
],
51+
"ContainerARN": "arn:aws:ecs:us-west-2:111122223333:container/05966557-f16c-49cb-9352-24b3a0dcd0e1",
52+
"LogOptions": {
53+
"awslogs-create-group": "true",
54+
"awslogs-group": "/ecs/containerlogs",
55+
"awslogs-region": "us-west-2",
56+
"awslogs-stream": "ecs/curl/cd189a933e5849daa93386466019ab50",
57+
},
58+
"LogDriver": "awslogs",
59+
}
60+
61+
test_task = {
62+
"Cluster": "arn:aws:ecs:us-west-2:111122223333:cluster/default",
63+
"TaskARN": "arn:aws:ecs:us-west-2:111122223333:task/default/e9028f8d5d8e4f258373e7b93ce9a3c3",
64+
"Family": "curltest",
65+
"Revision": "3",
66+
"DesiredStatus": "RUNNING",
67+
"KnownStatus": "RUNNING",
68+
"Limits": {"CPU": 0.25, "Memory": 512},
69+
"PullStartedAt": "2020-10-08T20:47:16.053330955Z",
70+
"PullStoppedAt": "2020-10-08T20:47:19.592684631Z",
71+
"AvailabilityZone": "us-west-2a",
72+
"Containers": [
73+
{
74+
"DockerId": "e9028f8d5d8e4f258373e7b93ce9a3c3-2495160603",
75+
"Name": "curl",
76+
"DockerName": "curl",
77+
"Image": "111122223333.dkr.ecr.us-west-2.amazonaws.com/curltest:latest",
78+
"ImageID": "sha256:25f3695bedfb454a50f12d127839a68ad3caf91e451c1da073db34c542c4d2cb",
79+
"Labels": {
80+
"com.amazonaws.ecs.cluster": "arn:aws:ecs:us-west-2:111122223333:cluster/default",
81+
"com.amazonaws.ecs.container-name": "curl",
82+
"com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/e9028f8d5d8e4f258373e7b93ce9a3c3",
83+
"com.amazonaws.ecs.task-definition-family": "curltest",
84+
"com.amazonaws.ecs.task-definition-version": "3",
85+
},
86+
"DesiredStatus": "RUNNING",
87+
"KnownStatus": "RUNNING",
88+
"Limits": {"CPU": 10, "Memory": 128},
89+
"CreatedAt": "2020-10-08T20:47:20.567813946Z",
90+
"StartedAt": "2020-10-08T20:47:20.567813946Z",
91+
"Type": "NORMAL",
92+
"Networks": [
93+
{
94+
"NetworkMode": "awsvpc",
95+
"IPv4Addresses": ["192.0.2.3"],
96+
"IPv6Addresses": ["2001:dB8:10b:1a00:32bf:a372:d80f:e958"],
97+
"AttachmentIndex": 0,
98+
"MACAddress": "02:b7:20:19:72:39",
99+
"IPv4SubnetCIDRBlock": "192.0.2.0/24",
100+
"IPv6SubnetCIDRBlock": "2600:1f13:10b:1a00::/64",
101+
"DomainNameServers": ["192.0.2.2"],
102+
"DomainNameSearchList": ["us-west-2.compute.internal"],
103+
"PrivateDNSName": "ip-172-31-30-173.us-west-2.compute.internal",
104+
"SubnetGatewayIpv4Address": "192.0.2.0/24",
105+
}
106+
],
107+
"ContainerARN": "arn:aws:ecs:us-west-2:111122223333:container/1bdcca8b-f905-4ee6-885c-4064cb70f6e6",
108+
"LogOptions": {
109+
"awslogs-create-group": "true",
110+
"awslogs-group": "/ecs/containerlogs",
111+
"awslogs-region": "us-west-2",
112+
"awslogs-stream": "ecs/curl/e9028f8d5d8e4f258373e7b93ce9a3c3",
113+
},
114+
"LogDriver": "awslogs",
115+
}
116+
],
117+
"LaunchType": "FARGATE",
118+
}

0 commit comments

Comments
 (0)