User-defined Custom Config (#267)

* User-defined Custom Config

 * Dict[str, Any] structure added
 * Fixed environment variable substitution pattern
 * Unit tests added

* User-defined Custom Config

 * default_factory instead of Optional

Author: igorsereda

src/dipdup/config.py

@@ -54,7 +54,7 @@
 from dipdup.utils import pascal_to_snake
 from dipdup.utils import snake_to_pascal
 
-ENV_VARIABLE_REGEX = r'\${([\w]*):-(.*)}'  # ${VARIABLE:-default}
+ENV_VARIABLE_REGEX = r'\$\{(?P<var_name>[\w]+)(?:\:\-(?P<default_value>.*))?\}'  # ${VARIABLE:-default} | ${VARIABLE}
 DEFAULT_RETRY_COUNT = 3
 DEFAULT_RETRY_SLEEP = 1
 DEFAULT_METADATA_URL = 'https://metadata.dipdup.net'
@@ -983,6 +983,7 @@ class DipDupConfig:
     :param jobs: Mapping of job aliases and job configs
     :param sentry: Sentry integration config
     :param advanced: Advanced config
+    :param custom: User-defined Custom config
     """
 
     spec_version: str
@@ -998,6 +999,7 @@ class DipDupConfig:
     sentry: Optional[SentryConfig] = None
     prometheus: Optional[PrometheusConfig] = None
     advanced: AdvancedConfig = AdvancedConfig()
+    custom: Dict[str, Any] = field(default_factory=dict)
 
     def __post_init_post_parse__(self):
         self.paths: List[str] = []
@@ -1058,12 +1060,12 @@ def load(
             if environment:
                 _logger.debug('Substituting environment variables')
                 for match in re.finditer(ENV_VARIABLE_REGEX, raw_config):
-                    variable, default_value = match.group(1), match.group(2)
-                    config_environment[variable] = default_value
-                    value = env.get(variable)
-                    if not default_value and not value:
+                    variable, default_value = match.group('var_name'), match.group('default_value')
+                    value = env.get(variable, default_value)
+                    if not value:
                         raise ConfigurationError(f'Environment variable `{variable}` is not set')
-                    placeholder = '${' + variable + ':-' + default_value + '}'
+                    config_environment[variable] = value
+                    placeholder = match.group(0)
                     raw_config = raw_config.replace(placeholder, value or default_value)
 
             json_config.update(yaml.load(raw_config))

tests/test_dipdup/test_config/test_custom_config.py

@@ -0,0 +1,109 @@
+import os
+from os.path import dirname
+from os.path import join
+
+import pytest
+from _pytest.tmpdir import TempPathFactory
+
+from dipdup.config import DipDupConfig
+from dipdup.exceptions import ConfigurationError
+
+
+class TestCustomConfig:
+    @pytest.fixture(scope="session")
+    def dummy_config_path(self) -> str:
+        return join(dirname(dirname(__file__)), 'dipdup.yml')
+
+    @staticmethod
+    def appended_config_path(dummy_config_path: str, tmp_path_factory: TempPathFactory, append_raw: str) -> str:
+        config_file = tmp_path_factory.mktemp('config') / 'dipdup.yml'
+        with open(dummy_config_path, 'r') as f:
+            config_raw = f.read()
+
+        with open(config_file, 'a') as f:
+            f.write(config_raw)
+            f.write(append_raw)
+
+        return str(config_file)
+
+    @pytest.fixture(
+        scope="session",
+        params=(
+            [
+                """
+custom:
+    foo: bar
+    spam:
+      - eggs
+      - rice
+
+"""
+            ]
+        ),
+    )
+    def config_with_custom_section_path(self, dummy_config_path: str, tmp_path_factory: TempPathFactory, request) -> str:
+        return self.appended_config_path(dummy_config_path, tmp_path_factory, request.param)
+
+    @staticmethod
+    def test_empty_custom_section(dummy_config_path: str):
+        config = DipDupConfig.load([dummy_config_path], False)
+        config.initialize(True)
+        assert hasattr(config, 'custom')
+        assert config.custom == {}
+
+    @staticmethod
+    def test_custom_section_items(config_with_custom_section_path: str):
+        config = DipDupConfig.load([config_with_custom_section_path], False)
+        config.initialize(True)
+
+        assert hasattr(config, 'custom')
+        assert isinstance(config.custom, dict)
+
+        assert config.custom['foo'] == 'bar'
+
+        spam = config.custom['spam']
+        assert isinstance(spam, list)
+        assert 'eggs' in spam
+        assert 'rice' in spam
+
+    @pytest.mark.parametrize(
+        'value, expected',
+        (
+            ('${USER:-dipdup}', os.environ.get('USER')),
+            ('${USER:-}', os.environ.get('USER')),
+            ('${USER}', os.environ.get('USER')),
+            ('${DEFINITELY_NOT_DEFINED:-default_value}', 'default_value'),
+            ('${DEFINITELY_NOT_DEFINED:- some_spaces_is_ok  }', 'some_spaces_is_ok'),
+        ),
+    )
+    def test_env_parsing_positive(self, value, expected, dummy_config_path, tmp_path_factory):
+        append_raw = f"""
+custom:
+    var_from_env: {value}
+"""
+        config_path = self.appended_config_path(dummy_config_path, tmp_path_factory, append_raw)
+        config = DipDupConfig.load([config_path], True)
+        config.initialize(True)
+
+        assert hasattr(config, 'custom')
+        assert isinstance(config.custom, dict)
+
+        assert config.custom['var_from_env'] == expected
+
+    @pytest.mark.parametrize(
+        'value',
+        (
+            '${DEFINITELY_NOT_DEFINED}',
+            '${DEFINITELY_NOT_DEFINED:-}',
+        ),
+    )
+    def test_env_parsing_negative(self, value, dummy_config_path, tmp_path_factory):
+        append_raw = f"""
+custom:
+    var_from_env: {value}
+"""
+        config_path = self.appended_config_path(dummy_config_path, tmp_path_factory, append_raw)
+        with pytest.raises(ConfigurationError) as exception_info:
+            DipDupConfig.load([config_path], True)
+
+        assert str(exception_info.value) == 'Environment variable `DEFINITELY_NOT_DEFINED` is not set'