Skip to content

Commit c4f1be0

Browse files
JohnPrestonJohnPreston
committed
Support for certbot-aws-store files
* Pull certificates from certbot-aws-files registry * Improved configuration import to job definition * Files are no longer mandatory
1 parent d1a1abb commit c4f1be0

File tree

15 files changed

+1079
-485
lines changed

15 files changed

+1079
-485
lines changed

‎Makefile‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -93,13 +93,13 @@ dist: clean ## builds source and wheel package
9393
poetry build
9494
ls -l dist
9595

96-
install: clean ## install the package to the active Python's site-packages
96+
install: clean conform ## install the package to the active Python's site-packages
9797
pip install . --use-pep517
9898

99-
conform : ## Conform to a standard of coding syntax
99+
conform: data-model
100100
isort --profile black ecs_files_composer
101101
black ecs_files_composer tests
102102
find ecs_files_composer -name "*.json" -type f -exec sed -i '1s/^\xEF\xBB\xBF//' {} +
103103

104104
data-model:
105-
datamodel-codegen
105+
datamodel-codegen

‎ecs-files-input.json‎

Lines changed: 81 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,6 @@
33
"type": "object",
44
"typeName": "input",
55
"description": "Configuration input definition for ECS Files Composer",
6-
"required": [
7-
"files"
8-
],
96
"properties": {
107
"files": {
118
"type": "object",
@@ -29,6 +26,15 @@
2926
}
3027
}
3128
},
29+
"certbot_store": {
30+
"type": "object",
31+
"uniqueItems": true,
32+
"patternProperties": {
33+
"[\\W\\-\\.]+$": {
34+
"$ref": "#/definitions/CertbotAwsStoreCertificate"
35+
}
36+
}
37+
},
3238
"IamOverride": {
3339
"type": "object",
3440
"$ref": "#/definitions/IamOverrideDef"
@@ -158,34 +164,47 @@
158164
"VersionStage": {
159165
"type": "string"
160166
},
167+
"JsonKey": {
168+
"type": "string",
169+
"description": "If the SecretString is a valid JSON, use the Key to map to the value stored in secret"
170+
},
161171
"IamOverride": {
162172
"$ref": "#/definitions/IamOverrideDef"
163173
}
164174
}
165175
},
166176
"S3Def": {
167-
"type": "object",
168-
"required": [
169-
"BucketName",
170-
"Key"
171-
],
172-
"properties": {
173-
"BucketName": {
174-
"type": "string",
175-
"description": "Name of the S3 Bucket"
176-
},
177-
"BucketRegion": {
178-
"type": "string",
179-
"description": "S3 Region to use. Default will ignore or retrieve via s3:GetBucketLocation"
177+
"oneOf": [
178+
{
179+
"type": "object",
180+
"required": [
181+
"BucketName",
182+
"Key"
183+
],
184+
"properties": {
185+
"BucketName": {
186+
"type": "string",
187+
"description": "Name of the S3 Bucket"
188+
},
189+
"BucketRegion": {
190+
"type": "string",
191+
"description": "S3 Region to use. Default will ignore or retrieve via s3:GetBucketLocation"
192+
},
193+
"Key": {
194+
"type": "string",
195+
"description": "Full path to the file to retrieve"
196+
},
197+
"IamOverride": {
198+
"$ref": "#/definitions/IamOverrideDef"
199+
}
200+
}
180201
},
181-
"Key": {
202+
{
182203
"type": "string",
183-
"description": "Full path to the file to retrieve"
184-
},
185-
"IamOverride": {
186-
"$ref": "#/definitions/IamOverrideDef"
204+
"pattern": "^arn:aws(?:-[a-z]+)?:s3:::(\\S+)::(\\S+)$",
205+
"description": "OneLiner with bucket ARN and path to key."
187206
}
188-
}
207+
]
189208
},
190209
"IamOverrideDef": {
191210
"type": "object",
@@ -290,6 +309,45 @@
290309
"default": "root"
291310
}
292311
}
312+
},
313+
"CertbotAwsStoreCertificate": {
314+
"type": "object",
315+
"required": [
316+
"storage_path"
317+
],
318+
"properties": {
319+
"storage_path": {
320+
"type": "string",
321+
"pattern": "^/[\\x00-\\x7F]+$",
322+
"description": "path to folder to store the certbot certificates into."
323+
},
324+
"table_name": {
325+
"type": "string",
326+
"description": "dynamodb table name of the certbot-aws-store registry",
327+
"default": "certbot-registry"
328+
},
329+
"table_region_name": {
330+
"type": "string",
331+
"description": "Region in which the table_name is. Defaults to profile default region, or eu-west-1"
332+
}
333+
}
334+
}
335+
},
336+
"anyOf": [
337+
{
338+
"required": [
339+
"files"
340+
]
341+
},
342+
{
343+
"required": [
344+
"certbot_store"
345+
]
346+
},
347+
{
348+
"required": [
349+
"certificates"
350+
]
293351
}
294-
}
352+
]
295353
}

‎ecs_files_composer/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# SPDX-License-Identifier: MPL-2.0
2-
# Copyright 2020-2021 John Mille<[email protected]>
2+
# Copyright 2020-2022 John Mille<[email protected]>
33

44
"""Top-level package for ECS Files Composer."""
55

‎ecs_files_composer/aws_mgmt.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# SPDX-License-Identifier: MPL-2.0
2-
# Copyright 2020-2021 John Mille<[email protected]>
2+
# Copyright 2020-2022 John Mille<[email protected]>
33

44
"""AWS module."""
55

‎ecs_files_composer/certbot_aws_store.py‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# SPDX-License-Identifier: MPL-2.0
2+
# Copyright 2020-2022 John Mille<[email protected]>
3+
4+
"""
5+
Manages certbot certificates download from certbot-aws-store
6+
"""
7+
8+
from __future__ import annotations
9+
10+
from typing import TYPE_CHECKING
11+
12+
if TYPE_CHECKING:
13+
from .input import Model
14+
15+
from certbot_aws_store.certificate import AcmeCertificate
16+
17+
from ecs_files_composer.common import LOG
18+
19+
20+
def handle_certbot_store_certificates(job: Model) -> None:
21+
if not job.certbot_store:
22+
return
23+
for _hostname, _definition in job.certbot_store.items():
24+
certificate = AcmeCertificate(
25+
_hostname,
26+
None,
27+
table_name=_definition.table_name if _definition.table_name else None,
28+
region_name=_definition.table_region_name
29+
if _definition.table_region_name
30+
else None,
31+
)
32+
try:
33+
certificate.pull(_definition.storage_path)
34+
LOG.info("Successfully pulled certificates for %s", _hostname)
35+
except Exception as error:
36+
print(error)
37+
print("Failed to download certificate from certbot-aws-store", _hostname)

‎ecs_files_composer/certificates_mgmt.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# SPDX-License-Identifier: MPL-2.0
2-
# Copyright 2020-2021 John Mille<[email protected]>
2+
# Copyright 2020-2022 John Mille<[email protected]>
33

44
import pathlib
55
import socket

‎ecs_files_composer/cli.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# SPDX-License-Identifier: MPL-2.0
2-
# Copyright 2020-2021 John Mille<[email protected]>
2+
# Copyright 2020-2022 John Mille<[email protected]>
33

44
"""Console script for ecs_files_composer."""
55
import argparse

‎ecs_files_composer/common.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# SPDX-License-Identifier: MPL-2.0
2-
# Copyright 2020-2021 John Mille<[email protected]>
2+
# Copyright 2020-2022 John Mille<[email protected]>
33

44
import logging as logthings
55
import sys

‎ecs_files_composer/ecs_files_composer.py‎

Lines changed: 28 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,15 @@
11
# SPDX-License-Identifier: MPL-2.0
2-
# Copyright 2020-2021 John Mille<[email protected]>
2+
# Copyright 2020-2022 John Mille<[email protected]>
33

44
"""Main module."""
55

6+
from __future__ import annotations
7+
8+
from typing import TYPE_CHECKING
9+
10+
if TYPE_CHECKING:
11+
from .input import Model
12+
613
import json
714
import uuid
815
from os import environ, path
@@ -13,6 +20,7 @@
1320

1421
from ecs_files_composer import input
1522
from ecs_files_composer.aws_mgmt import S3Fetcher, SecretFetcher, SsmFetcher
23+
from ecs_files_composer.certbot_aws_store import handle_certbot_store_certificates
1624
from ecs_files_composer.certificates_mgmt import process_x509_certs
1725
from ecs_files_composer.common import LOG
1826
from ecs_files_composer.files_mgmt import File
@@ -110,23 +118,29 @@ def init_config(
110118
raise
111119

112120

113-
def start_jobs(config, override_session=None):
121+
def process_files(job: Model, override_session=None) -> None:
122+
files: list = []
123+
for file_path, file in job.files.items():
124+
file_redef = File(**file.dict())
125+
file_redef.path = file_path
126+
files.append(file_redef)
127+
for file in files:
128+
file.handler(job.iam_override, override_session)
129+
LOG.info(f"Tasks for {file.path} completed.")
130+
131+
132+
def start_jobs(config: dict, override_session=None):
114133
"""
115134
Starting point to run the files job
116135
117136
:param config:
118137
:param override_session:
119138
:return:
120139
"""
121-
if not keyisset("files", config):
122-
raise KeyError("Missing required files from configuration input")
123-
job = input.Model(files=config["files"]).parse_obj(config)
124-
process_x509_certs(job)
125-
for file_path, file in job.files.items():
126-
if not isinstance(file, File):
127-
file_def = File().parse_obj(file)
128-
job.files[file_path] = file_def
129-
file_def.path = file_path
130-
for file in job.files.values():
131-
file.handler(job.iam_override, override_session)
132-
LOG.info(f"Tasks for {file.path} completed.")
140+
job = input.Model(**config)
141+
if job.certificates:
142+
process_x509_certs(job)
143+
if job.certbot_store:
144+
handle_certbot_store_certificates(job)
145+
if job.files:
146+
process_files(job, override_session)

‎ecs_files_composer/envsubst.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# SPDX-License-Identifier: MPL-2.0
2-
# Copyright 2020-2021 John Mille <[email protected]>
2+
# Copyright 2020-2022 John Mille <[email protected]>
33

44
"""
55
Module to do a better env variables handling.

0 commit comments

Comments
 (0)