Declare yaml as a dependency with extensions

At the same time, the syntax for the !ENV and !ENVFILE tags is changed
to just the env-var name. No need for ${} or $().

Signed-off-by: Ivan Kanakarakis <[email protected]>

Author: c00kiemon5ter

doc/README.md

@@ -34,16 +34,16 @@ SATOSA is configured using YAML.
 All default configuration files, as well as an example WSGI application for the proxy, can be found
 in the [example directory](../example).
 
-A configuration value that includes the tag !ENV will have a value of the form `${SOME_ENVIRONMENT_VARIABLE}`
+A configuration value that includes the tag !ENV will have a value of the form `SOME_ENVIRONMENT_VARIABLE`
 replaced with the value from the process environment variable of the same name. For example if the file
 `ldap_attribute_store.yaml' includes
 
 ```
-bind_password: !ENV ${LDAP_BIND_PASSWORD}
+bind_password: !ENV LDAP_BIND_PASSWORD
 ```
 
 and the SATOSA process environment includes the environment variable `LDAP_BIND_PASSWORD` with
-value `my_password` then the configuration for `bind_password` will be `my_password`.
+value `my_password` then the configuration value for `bind_password` will be `my_password`.
 
 
 ## <a name="proxy_conf" style="color:#000000">SATOSA proxy configuration</a>: `proxy_conf.yaml.example`

example/plugins/microservices/ldap_attribute_store.yaml.example

@@ -9,10 +9,10 @@ config:
       ldap_url: ldaps://ldap.example.org
       bind_dn: cn=admin,dc=example,dc=org
       # Obtain bind password from environment variable LDAP_BIND_PASSWORD.
-      bind_password: !ENV ${LDAP_BIND_PASSWORD}
+      bind_password: !ENV LDAP_BIND_PASSWORD
       # Obtain bind password from file pointed to by
       # environment variable LDAP_BIND_PASSWORD_FILE.
-      # bind_password: !ENVFILE $(LDAP_BIND_PASSWORD)
+      # bind_password: !ENVFILE LDAP_BIND_PASSWORD
       search_base: ou=People,dc=example,dc=org
       read_only: true
       auto_bind: true

src/satosa/satosa_config.py

@@ -4,11 +4,11 @@
 import logging
 import os
 import os.path
-import re
 
-import yaml
+from satosa.exception import SATOSAConfigurationError
+from satosa.yaml import load as yaml_load
+from satosa.yaml import YAMLError
 
-from .exception import SATOSAConfigurationError
 
 logger = logging.getLogger(__name__)
 
@@ -145,69 +145,11 @@ def _load_yaml(self, config_file):
         :param config_file: config to load. Can be file path or yaml string
         :return: Loaded config
         """
-        # Tag to indicate environment variable: !ENV
-        tag_env = '!ENV'
-
-        # Pattern for environment variable: ${word}
-        pattern_env = re.compile('.*?\${(\w+)}.*?')
-
-        yaml.SafeLoader.add_implicit_resolver(tag_env, pattern_env, None)
-
-        def constructor_env_variables(loader, node):
-            """
-            Extracts the environment variable from the node's value.
-            :param yaml.Loader loader: the yaml loader
-            :param node: the current node in the yaml
-            :return: value of the environment variable
-            """
-            value = loader.construct_scalar(node)
-            match = pattern_env.findall(value)
-            if match:
-                new_value = value
-                for m in match:
-                    new_value = new_value.replace('${' + m + '}',
-                                                  os.environ.get(m, m))
-                return new_value
-            return value
-
-        yaml.SafeLoader.add_constructor(tag_env, constructor_env_variables)
-
-        # Tag to indicate file pointed to by environment variable: !ENVFILE
-        tag_env_file = '!ENVFILE'
-
-        # Pattern for environment variable: $(word)
-        pattern_env_file = re.compile('.*?\$\((\w+)\).*?')
-
-        yaml.SafeLoader.add_implicit_resolver(tag_env_file,
-                                              pattern_env_file, None)
-
-        def constructor_envfile_variables(loader, node):
-            """
-            Extracts the environment variable from the node's value.
-            :param yaml.Loader loader: the yaml loader
-            :param node: the current node in the yaml
-            :return: value read from file pointed to by environment variable
-            """
-            value = loader.construct_scalar(node)
-            match = pattern_env_file.findall(value)
-            if match:
-                new_value = value
-                for m in match:
-                    path = os.environ.get(m, '')
-                    if os.path.exists(path):
-                        with open(path, 'r') as f:
-                            new_value = new_value.replace('$(' + m + ')',
-                                                          f.read().strip())
-                return new_value
-            return value
-
-        yaml.SafeLoader.add_constructor(tag_env_file,
-                                        constructor_envfile_variables)
 
         try:
             with open(os.path.abspath(config_file)) as f:
-                return yaml.safe_load(f.read())
-        except yaml.YAMLError as exc:
+                return yaml_load(f.read())
+        except YAMLError as exc:
             logger.error("Could not parse config as YAML: {}".format(exc))
             if hasattr(exc, 'problem_mark'):
                 mark = exc.problem_mark

src/satosa/yaml.py

@@ -0,0 +1,58 @@
+import os
+import re
+
+from yaml import SafeLoader as _safe_loader
+from yaml import YAMLError
+from yaml import safe_load as load
+
+
+def _constructor_env_variables(loader, node):
+    """
+    Extracts the environment variable from the node's value.
+    :param yaml.Loader loader: the yaml loader
+    :param node: the current node in the yaml
+    :return: value of the environment variable
+    """
+    raw_value = loader.construct_scalar(node)
+    new_value = os.environ.get(raw_value)
+    if new_value is None:
+        msg = "Cannot construct value from {node}: {value}".format(
+            node=node, value=new_value
+        )
+        raise YAMLError(msg)
+    return new_value
+
+
+def _constructor_envfile_variables(loader, node):
+    """
+    Extracts the environment variable from the node's value.
+    :param yaml.Loader loader: the yaml loader
+    :param node: the current node in the yaml
+    :return: value read from file pointed to by environment variable
+    """
+    raw_value = loader.construct_scalar(node)
+    filepath = os.environ.get(raw_value)
+    if filepath is None:
+        msg = "Cannot construct value from {node}: {path}".format(
+            node=node, path=filepath
+        )
+        raise YAMLError(msg)
+
+    try:
+        with open(filepath, "r") as fd:
+            new_value = fd.read()
+    except (TypeError, IOError) as e:
+        msg = "Cannot construct value from {node}: {path}".format(
+            node=node, path=filepath
+        )
+        raise YAMLError(msg) from e
+    else:
+        return new_value
+
+
+TAG_ENV = "!ENV"
+TAG_ENV_FILE = "!ENVFILE"
+
+
+_safe_loader.add_constructor(TAG_ENV, _constructor_env_variables)
+_safe_loader.add_constructor(TAG_ENV_FILE, _constructor_envfile_variables)

tests/satosa/test_satosa_config.py

@@ -78,16 +78,19 @@ def test_can_read_endpoint_configs_from_file(self, satosa_config_dict, modules_k
 
     def test_can_substitute_from_environment_variable(self, monkeypatch):
         monkeypatch.setenv("SATOSA_COOKIE_STATE_NAME", "oatmeal_raisin")
-        config = SATOSAConfig(os.path.join(TEST_RESOURCE_BASE_PATH,
-                              "proxy_conf_environment_test.yaml"))
+        config = SATOSAConfig(
+            os.path.join(TEST_RESOURCE_BASE_PATH, "proxy_conf_environment_test.yaml")
+        )
 
         assert config["COOKIE_STATE_NAME"] == 'oatmeal_raisin'
 
     def test_can_substitute_from_environment_variable_file(self, monkeypatch):
-        cookie_file = os.path.join(TEST_RESOURCE_BASE_PATH,
-                                   'cookie_state_name')
+        cookie_file = os.path.join(TEST_RESOURCE_BASE_PATH, 'cookie_state_name')
         monkeypatch.setenv("SATOSA_COOKIE_STATE_NAME_FILE", cookie_file)
-        config = SATOSAConfig(os.path.join(TEST_RESOURCE_BASE_PATH,
-                              "proxy_conf_environment_file_test.yaml"))
+        config = SATOSAConfig(
+            os.path.join(
+                TEST_RESOURCE_BASE_PATH, "proxy_conf_environment_file_test.yaml"
+            )
+        )
 
         assert config["COOKIE_STATE_NAME"] == 'chocolate_chip'

tests/test_resources/cookie_state_name

@@ -1 +1 @@
-chocolate_chip
+chocolate_chip

tests/test_resources/proxy_conf_environment_file_test.yaml

@@ -4,7 +4,7 @@ STATE_ENCRYPTION_KEY: state_encryption_key
 
 INTERNAL_ATTRIBUTES: {"attributes": {}}
 
-COOKIE_STATE_NAME: !ENVFILE $(SATOSA_COOKIE_STATE_NAME_FILE)
+COOKIE_STATE_NAME: !ENVFILE SATOSA_COOKIE_STATE_NAME_FILE
 
 BACKEND_MODULES: []
 FRONTEND_MODULES: []

tests/test_resources/proxy_conf_environment_test.yaml

@@ -4,7 +4,7 @@ STATE_ENCRYPTION_KEY: state_encryption_key
 
 INTERNAL_ATTRIBUTES: {"attributes": {}}
 
-COOKIE_STATE_NAME: !ENV ${SATOSA_COOKIE_STATE_NAME}
+COOKIE_STATE_NAME: !ENV SATOSA_COOKIE_STATE_NAME
 
 BACKEND_MODULES: []
 FRONTEND_MODULES: []