feat: SOPS secrets support

Signed-off-by: Aurora Gaffney <agaffney@applause.com>

Author: agaffney

deploy_config_generator/__main__.py

@@ -16,6 +16,7 @@
 from deploy_config_generator.template import Template
 from deploy_config_generator.errors import DeployConfigError, DeployConfigGenerationError, ConfigError, VarsReplacementError
 from deploy_config_generator.utils import yaml_dump, show_traceback
+from deploy_config_generator.secrets import Secrets, SecretsException
 
 DISPLAY = None
 SITE_CONFIG = None
@@ -78,7 +79,7 @@ def load_vars_files(varset, vars_dir, patterns, allow_var_references=True):
         varset.read_vars_file(vars_file, allow_var_references=allow_var_references)
 
 
-def load_output_plugins(varset, output_dir, config_version):
+def load_output_plugins(varset, secrets, output_dir, config_version):
     '''
     Find, import, and instantiate all output plugins
     '''
@@ -111,7 +112,7 @@ def load_output_plugins(varset, output_dir, config_version):
                     raise Exception('name specified in OutputPlugin class (%s) does not match file name (%s)' % (cls.NAME, name))
                 DISPLAY.v('Loading plugin %s' % name)
                 # Instantiate plugin class
-                plugins.append(cls(varset, output_dir, config_version))
+                plugins.append(cls(varset, secrets, output_dir, config_version))
             except ConfigError as e:
                 DISPLAY.display('Plugin configuration error: %s: %s' % (name, str(e)))
                 sys.exit(1)
@@ -263,8 +264,24 @@ def main():
     DISPLAY.vvvv()
     DISPLAY.vvvv(yaml_dump(dict(varset), indent=2))
 
+    secrets = Secrets()
+
+    try:
+        for f in SITE_CONFIG.secrets_files:
+            secrets.load_secrets_file(f)
+    except SecretsException as e:
+        DISPLAY.display('Error loading secrets file %s' % (e.path,))
+        DISPLAY.v('Stderr:')
+        DISPLAY.v()
+        DISPLAY.v(e.stderr)
+        sys.exit(1)
+
+    DISPLAY.vvvv('Secrets:')
+    DISPLAY.vvvv()
+    DISPLAY.vvvv(yaml_dump(dict(secrets), indent=2))
+
     try:
-        deploy_config = DeployConfig(os.path.join(deploy_dir, SITE_CONFIG.deploy_config_file), varset)
+        deploy_config = DeployConfig(os.path.join(deploy_dir, SITE_CONFIG.deploy_config_file), varset, secrets)
         deploy_config.set_config(varset.replace_vars(deploy_config.get_config()))
     except DeployConfigError as e:
         DISPLAY.display('Error loading deploy config: %s' % str(e))
@@ -280,7 +297,7 @@ def main():
     DISPLAY.vvvv(yaml_dump(deploy_config.get_config(), indent=2))
 
     deploy_config_version = deploy_config.get_version() or SITE_CONFIG.default_config_version
-    output_plugins = load_output_plugins(varset, args.output_dir, deploy_config_version)
+    output_plugins = load_output_plugins(varset, secrets, args.output_dir, deploy_config_version)
 
     DISPLAY.vvv('Available output plugins:')
     DISPLAY.vvv()

deploy_config_generator/deploy_config.py

@@ -12,8 +12,9 @@ class DeployConfig(object):
     _path = None
     _version = None
 
-    def __init__(self, path, varset):
+    def __init__(self, path, varset, secrets):
         self._vars = varset
+        self._secrets = secrets
         self._display = Display()
         self.load(path)
 
@@ -68,7 +69,7 @@ def apply_default_apps(self, default_apps):
                     condition = app[CONDITION_KEY]
                     del app[CONDITION_KEY]
                     template = Template()
-                    tmp_vars = dict(VARS=dict(self._vars))
+                    tmp_vars = dict(VARS=dict(self._vars), SECRETS=dict(self._secrets))
                     # Continue to next item if we have a condition and it evaluated to False
                     if not template.evaluate_condition(condition, tmp_vars):
                         continue

deploy_config_generator/output/__init__.py

@@ -30,8 +30,9 @@ class OutputPluginBase(object):
     )
     PRIORITY = 1
 
-    def __init__(self, varset, output_dir, config_version):
+    def __init__(self, varset, secrets, output_dir, config_version):
         self._vars = varset
+        self._secrets = secrets
         self._output_dir = output_dir
         self._display = Display()
         self._template = Template()
@@ -201,6 +202,8 @@ def build_app_vars(self, index, app, path=''):
             'APP': self.merge_with_field_defaults(app),
             # Parsed vars
             'VARS': dict(self._vars),
+            # Secrets
+            'SECRETS': dict(self._secrets),
         }
         return app_vars
 

deploy_config_generator/secrets.py

@@ -0,0 +1,28 @@
+import subprocess
+
+from deploy_config_generator.utils import yaml_load
+
+SOPS_BINARY = 'sops'
+
+
+class SecretsException(Exception):
+
+    def __init__(self, path, stderr):
+        self.path = path
+        self.stderr = stderr
+        super().__init__('Failed to decrypt secrets')
+
+
+class Secrets(dict):
+
+    '''
+    This class manages a set of SOPS-encrypted secrets
+    '''
+
+    def load_secrets_file(self, path):
+        cp = subprocess.run([SOPS_BINARY, 'decrypt', '--output-type=yaml', path], capture_output=True)
+        if cp.returncode != 0:
+            raise SecretsException(path, cp.stderr)
+        # Parse decrypted YAML from SOPS
+        data = yaml_load(cp.stdout)
+        self.update(data)

deploy_config_generator/site_config.py

@@ -49,6 +49,8 @@ class SiteConfig(object, metaclass=Singleton):
         'default_apps': {},
         # Default vars
         'default_vars': {},
+        # SOPS secrets files to load
+        'secrets_files': [],
     }
 
     def __init__(self, env=None):

setup.py

@@ -53,7 +53,7 @@ def run(self):
 
 setup(
     name='deploy-config-generator',
-    version='2.29.0',
+    version='2.30.0',
     url='https://github.com/ApplauseOSS/deploy-config-generator',
     license='MIT',
     description='Utility to generate service deploy configurations',

tests/integration/age-key.txt

@@ -0,0 +1,6 @@
+# NOTICE: this key file was created for use in the integration tests and should not be
+# used elsewhere
+#
+# created: 2025-11-04T20:43:20Z
+# public key: age1cnlgtr6w8p8phdxqevklllex0u7y97574xclnf0qv050emsstgpqv23h0q
+AGE-SECRET-KEY-1D4HH6NNHZZSRM0WD23PEWT9FTWR2708FNDV957WPTJ0CE0WDFG9QY3QUSY

tests/integration/secrets/deploy/config.yml

@@ -0,0 +1,6 @@
+---
+test:
+  dummy: True
+  parent1:
+    - child2_1: '{{ SECRETS.foo }}'
+      child2_2: '{{ SECRETS.some }}'

tests/integration/secrets/expected_output/dummy-001.foo

@@ -0,0 +1,14 @@
+Dummy output plugin
+
+App config:
+
+{
+  "dummy": true,
+  "parent1": [
+    {
+      "child2_1": "bar",
+      "child2_2": "value",
+      "parent2": []
+    }
+  ]
+}

tests/integration/secrets/runme.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+TEST_DIR=$(dirname $0)
+cd ${TEST_DIR}
+
+export PYTHONPATH=../../..
+
+export SOPS_AGE_KEY_FILE=../age-key.txt
+
+set -e
+
+rm -rf tmp
+mkdir tmp
+
+set -x
+
+python3 -m deploy_config_generator -v -c site_config.yml -o tmp . $@
+
+diff -BurN expected_output tmp