Improved logging for extra fields, fixed some bugs and added new one

Author: shadowy-pycoder

README.md

@@ -54,6 +54,7 @@ redis:
 database:
   username: username
   password: password
+  name: db
 ```
 
 Import configuration files in your Python code with `pyya`:
@@ -67,18 +68,22 @@ logger.setLevel(logging.INFO)
 
 config = init_config(
     'config.yaml', 'default.config.yaml',
-    merge_configs = True,
-    sections_ignored_on_merge = ['redis'], # do not include redis in your config
     convert_keys_to_snake_case = False,
     add_underscore_prefix_to_keywords = False,
     raise_error_non_identifiers = False,
+    merge_configs = True,
+    sections_ignored_on_merge = ['redis'], # do not include redis in your config
     validate_data_types = True,
     allow_extra_sections = True,
     warn_extra_sections = True,
     )
 print(json.dumps(config))
 
 # Output:
+# 2025-09-05 09:13:17,280         WARNING         pyya            The following extra sections will be ignored:
+# {'database.name': 'db'}
+# 2025-09-05 09:13:17,281         INFO            pyya            The following sections were overwritten:
+# {database: {'host': 'localhost', 'port': 5432}}
 # {database: {"host": "localhost", "port": 5432, "username": "username", "password": "password"}}
 
 ```
@@ -89,19 +94,6 @@ Under the hood `pyya` uses [PyYAML](https://pypi.org/project/PyYAML/) to parse Y
 
 ### Flags
 
-```python
-# merge default and production configuration files
-# setting to `False` disables other flags and makes default config optional
-# `False` means "open config file and apply `yaml.safe_load` and `munchify` with no formatting"
-merge_configs=True
-```
-
-```python
-# list of sections to ignore when merging configs
-# it is useful when you have examples in your default config but do not want to have in the main one
-sections_ignored_on_merge=None
-```
-
 ```python
 # convert `camelCase` or `PascalCase` keys to `snake_case`
 convert_keys_to_snake_case=False
@@ -118,18 +110,31 @@ raise_error_non_identifiers=False
 ```
 
 ```python
-# raise error if data types in config are not the same as default (makes sense only if merge is enabled)
+# merge default and production configuration files
+# setting to `False` disables below flags and makes default config optional
+# `False` means "open config file and apply `yaml.safe_load` and `munchify` with specified formatting"
+merge_configs=True
+```
+
+```python
+# list of sections to ignore when merging configs
+# it is useful when you have examples in your default config but do not want to have in the main one
+sections_ignored_on_merge=None
+```
+
+```python
+# raise error if data types in production config are not the same as default
 # validation based on data types inferred from default config
 validate_data_types=True
 ```
 
 ```python
-# raise error if there are extra sections in config (may break if section name formatting is enabled)
+# raise error on any extra sections in production config
 allow_extra_sections=True
 ```
 
 ```python
-# warn about extra keys and values
+# if extra sections are allowed, warn about extra keys and values
 warn_extra_sections=True
 ```
 

pyproject.toml

@@ -38,7 +38,6 @@ python_version = "3.8"
 cache_dir = ".mypy_cache/strict"
 allow_redefinition = true
 strict_optional = false
-show_error_codes = true
 show_column_numbers = true
 warn_no_return = true
 disallow_any_unimported = false

pyya.egg-info/PKG-INFO

@@ -81,6 +81,7 @@ redis:
 database:
   username: username
   password: password
+  name: db
 ```
 
 Import configuration files in your Python code with `pyya`:
@@ -94,18 +95,22 @@ logger.setLevel(logging.INFO)
 
 config = init_config(
     'config.yaml', 'default.config.yaml',
-    merge_configs = True,
-    sections_ignored_on_merge = ['redis'], # do not include redis in your config
     convert_keys_to_snake_case = False,
     add_underscore_prefix_to_keywords = False,
     raise_error_non_identifiers = False,
+    merge_configs = True,
+    sections_ignored_on_merge = ['redis'], # do not include redis in your config
     validate_data_types = True,
     allow_extra_sections = True,
     warn_extra_sections = True,
     )
 print(json.dumps(config))
 
 # Output:
+# 2025-09-05 09:13:17,280         WARNING         pyya            The following extra sections will be ignored:
+# {'database.name': 'db'}
+# 2025-09-05 09:13:17,281         INFO            pyya            The following sections were overwritten:
+# {database: {'host': 'localhost', 'port': 5432}}
 # {database: {"host": "localhost", "port": 5432, "username": "username", "password": "password"}}
 
 ```
@@ -116,19 +121,6 @@ Under the hood `pyya` uses [PyYAML](https://pypi.org/project/PyYAML/) to parse Y
 
 ### Flags
 
-```python
-# merge default and production configuration files
-# setting to `False` disables other flags and makes default config optional
-# `False` means "open config file and apply `yaml.safe_load` and `munchify` with no formatting"
-merge_configs=True
-```
-
-```python
-# list of sections to ignore when merging configs
-# it is useful when you have examples in your default config but do not want to have in the main one
-sections_ignored_on_merge=None
-```
-
 ```python
 # convert `camelCase` or `PascalCase` keys to `snake_case`
 convert_keys_to_snake_case=False
@@ -145,18 +137,31 @@ raise_error_non_identifiers=False
 ```
 
 ```python
-# raise error if data types in config are not the same as default (makes sense only if merge is enabled)
+# merge default and production configuration files
+# setting to `False` disables below flags and makes default config optional
+# `False` means "open config file and apply `yaml.safe_load` and `munchify` with specified formatting"
+merge_configs=True
+```
+
+```python
+# list of sections to ignore when merging configs
+# it is useful when you have examples in your default config but do not want to have in the main one
+sections_ignored_on_merge=None
+```
+
+```python
+# raise error if data types in production config are not the same as default
 # validation based on data types inferred from default config
 validate_data_types=True
 ```
 
 ```python
-# raise error if there are extra sections in config (may break if section name formatting is enabled)
+# raise error on any extra sections in production config
 allow_extra_sections=True
 ```
 
 ```python
-# warn about extra keys and values
+# if extra sections are allowed, warn about extra keys and values
 warn_extra_sections=True
 ```
 

pyya.egg-info/SOURCES.txt

@@ -2,6 +2,7 @@ LICENSE
 README.md
 pyproject.toml
 pyya/__init__.py
+pyya/py.typed
 pyya.egg-info/PKG-INFO
 pyya.egg-info/SOURCES.txt
 pyya.egg-info/dependency_links.txt

pyya/__init__.py

@@ -29,28 +29,28 @@ def init_config(
     config: Union[str, Path] = 'config.yaml',
     default_config: Union[str, Path] = 'default.config.yaml',
     *,
-    merge_configs: bool = True,
-    sections_ignored_on_merge: Optional[List[str]] = None,
     convert_keys_to_snake_case: bool = False,
     add_underscore_prefix_to_keywords: bool = False,
     raise_error_non_identifiers: bool = False,
+    merge_configs: bool = True,
+    sections_ignored_on_merge: Optional[List[str]] = None,
     validate_data_types: bool = True,
     allow_extra_sections: bool = True,
     warn_extra_sections: bool = True,
 ) -> PyyaConfig:
     """Initialize attribute-stylish configuration from YAML file.
 
     Args:
-        config: path to config file
+        config: path to production config file
         default_config: path to default config file
-        merge_configs: merge default config with config (setting to `False` disables other flags)
-        sections_ignored_on_merge: list of sections to ignore when merging configs
         convert_keys_to_snake_case: convert config section names to snake case
         add_underscore_prefix_to_keywords: add underscore prefix to Python keywords
         raise_error_non_identifiers: raise error if config section name is not a valid identifier
-        validate_data_types: raise error if data types in config are not the same as default (makes sense only if merge is enabled)
-        allow_extra_sections: raise error if there are extra sections in config (may break if section name formatting is enabled)
-        warn_extra_sections: warn about extra keys and values
+        merge_configs: merge default config with production config (setting to `False` disables flags below)
+        sections_ignored_on_merge: list of sections to ignore when merging configs
+        validate_data_types: raise error if data types in production config are not the same as default
+        allow_extra_sections: raise error on any extra sections in production config
+        warn_extra_sections: if extra sections are allowed, warn about extra keys and values
     """
 
     def _merge_configs(
@@ -59,6 +59,7 @@ def _merge_configs(
         if sections is None:
             sections = []  # for logging
         for section, entry in _default_raw_data.items():
+            sections.append(section)
             if sections_ignored_on_merge:
                 if section in sections_ignored_on_merge:
                     logger.debug(f'section `{section}` ignored on merge')
@@ -67,26 +68,17 @@ def _merge_configs(
                     # is it fine to proccess already poped dicts on recursion?
                     entry = _pop_ignored_keys(entry)
             if section not in _raw_data or _raw_data[section] is None:
-                f_section = _sanitize_section(section)
-                sections.append(f_section)
-                if f_section not in _raw_data:
-                    if isinstance(entry, Dict):
-                        entry = _sanitize_keys(entry)
-                    _raw_data[f_section] = entry
+                if section not in _raw_data:
+                    _raw_data[section] = entry
                     logger.debug(f'section `{".".join(sections)}` with value `{entry}` taken from {default_config}')
                 else:
                     logger.debug(f'section `{".".join(sections)}` already exists in {config}, skipping')
             elif isinstance(entry, Dict):
-                sections.append(section)
                 _merge_configs(_raw_data[section], entry, sections)
-                f_section = _sanitize_section(section)
-                _raw_data[f_section] = _raw_data.pop(section, None)
             # TODO: add support for merging lists
             else:
-                f_section = _sanitize_section(section)
-                sections.append(f_section)
-                if f_section not in _raw_data:
-                    _raw_data[f_section] = _raw_data.pop(section, None)
+                if section not in _raw_data:
+                    _raw_data[section] = _raw_data.pop(section, None)
                 else:
                     logger.debug(f'section `{".".join(sections)}` already exists in {config}, skipping')
             sections.pop()
@@ -114,26 +106,28 @@ def _pop_ignored_keys(data: ConfigType) -> ConfigType:
         return data
 
     def _sanitize_keys(data: ConfigType) -> ConfigType:
-        for key, entry in data.copy().items():
+        for key in data.copy():
+            entry = data.pop(key, None)
+            key = _sanitize_section(key)
             if isinstance(entry, Dict):
-                _sanitize_keys(entry)
+                data[key] = _sanitize_keys(entry)
             else:
-                data[_sanitize_section(key)] = data.pop(key, None)
+                data[key] = entry
         return data
 
     def _pop_nested(d: Dict[str, Any], dotted_key: str, default: Any = None) -> Any:
         keys = dotted_key.split('.')
         current = d
 
         for k in keys[:-1]:
-            if not isinstance(current, dict) or k not in current:
+            if not isinstance(current, Dict) or k not in current:
                 return default
             current = current[k]
 
         return current.pop(keys[-1], default)
 
     # https://stackoverflow.com/questions/73958753/return-all-extra-passed-to-pydantic-model
-    class NewBase(BaseModel):
+    class ExtraBase(BaseModel):
         model_config = ConfigDict(strict=True, extra='allow' if allow_extra_sections else 'forbid')
         extra: Dict[str, Any] = Field(default={}, exclude=True)
 
@@ -155,15 +149,14 @@ def validator(cls, values: Any) -> Any:
         def extra_flat(self) -> Any:
             extra_flat = {**self.extra}
             for name, value in self:
-                if isinstance(value, NewBase):
+                if isinstance(value, ExtraBase):
                     data = {f'{name}.{k}': v for k, v in value.extra_flat.items()}
                     extra_flat.update(data)
             return extra_flat
 
     def _model_from_dict(name: str, data: Dict[str, Any]) -> Type[BaseModel]:
         fields: Dict[Any, Any] = {}
         for section, entry in data.items():
-            section = _sanitize_section(section)
             if isinstance(entry, Dict):
                 nested_model = _model_from_dict(section, entry)
                 fields[section] = (nested_model, entry)
@@ -178,12 +171,12 @@ def _model_from_dict(name: str, data: Dict[str, Any]) -> Type[BaseModel]:
                 fields[section] = (List[Any], entry)
             else:
                 fields[section] = (type(entry), entry)
-        model = create_model(name, **fields, __base__=NewBase)
-        return model
+        return create_model(name, **fields, __base__=ExtraBase)
 
     try:
         with open(Path(config)) as fstream:
             _raw_data: ConfigType = _yaml.safe_load(fstream) or {}
+            _raw_data = _sanitize_keys(_raw_data)
     except _yaml.YAMLError as e:
         err_msg = f'{config} file is corrupted: {e}'
         logger.error(err_msg)
@@ -193,10 +186,19 @@ def _model_from_dict(name: str, data: Dict[str, Any]) -> Type[BaseModel]:
         _raw_data = {}
 
     if merge_configs:
+        if sections_ignored_on_merge is None:
+            sections_ignored_on_merge = []
+        try:
+            sections_ignored_on_merge = [_sanitize_section(s) for s in sections_ignored_on_merge]
+        except Exception as e:
+            err_msg = f'Failed parsing `sections_ignored_on_merge`: {e!r}'
+            logger.error(err_msg)
+            raise PyyaError(err_msg) from None
         try:
             try:
                 with open(Path(default_config)) as fstream:
                     _default_raw_data: Optional[ConfigType] = _yaml.safe_load(fstream)
+                    _default_raw_data = _sanitize_keys(_default_raw_data)
             except _yaml.YAMLError as e:
                 err_msg = f'{default_config} file is corrupted: {e}'
                 logger.error(err_msg)
@@ -209,16 +211,14 @@ def _model_from_dict(name: str, data: Dict[str, Any]) -> Type[BaseModel]:
         # create copy for logging (only overwritten fields)
         _raw_data_copy = deepcopy(_raw_data)
         _merge_configs(_raw_data, _default_raw_data)
-        logger.debug(f'\n\nResulting config after merge:\n\n{pformat(_raw_data)}')
+        logger.debug(f'Resulting config after merge:\n{pformat(_raw_data)}')
         if validate_data_types:
             ConfigModel = _model_from_dict('ConfigModel', _default_raw_data)
             try:
                 validated_raw_data = ConfigModel.model_validate(_raw_data)
                 if extra_sections := validated_raw_data.extra_flat:  # type: ignore
                     if warn_extra_sections:
-                        logger.warning(
-                            f'\n\nThe following extra sections will be ignored:\n\n{pformat(extra_sections)}'
-                        )
+                        logger.warning(f'The following extra sections will be ignored:\n{pformat(extra_sections)}')
                     # remove extra sections from resulting config
                     for k in extra_sections:
                         _pop_nested(_raw_data_copy, k)
@@ -227,18 +227,11 @@ def _model_from_dict(name: str, data: Dict[str, Any]) -> Type[BaseModel]:
                 err_msg = f'Failed validating config file: {e!r}'
                 logger.error(err_msg)
                 raise PyyaError(err_msg) from None
-        # replace formatted sections in the copy of the config for logging
-        for k in _raw_data_copy.copy():
-            sk = _sanitize_section(k)
-            if sk in _raw_data:
-                _raw_data_copy.pop(k, None)
-                _raw_data_copy[sk] = _raw_data[sk]
         if _raw_data_copy:
-            logger.info(f'\n\nThe following sections were overwritten:\n\n{pformat(_raw_data_copy)}')
+            logger.info(f'The following sections were overwritten:\n{pformat(_raw_data_copy)}')
     try:
-        raw_data = _munchify(_raw_data)
-        logger.debug(f'\n\nResulting config:\n\n{pformat(raw_data)}')
-        return raw_data
+        logger.debug(f'Resulting config:\n{pformat(_raw_data)}')
+        return _munchify(_raw_data)
     except Exception as e:
         err_msg = f'Failed parsing config file: {e!r}'
         logger.error(err_msg)