config: Allow reading from multiple files

RELEASE_NOTES.md

@@ -6,11 +6,18 @@
 
 ## Upgrading
 
-<!-- Here goes notes on how to upgrade from previous versions, including deprecations and what they should be replaced with -->
+- The `ConfigManagingActor` now takes multiple configuration files as input, and the argument was renamed from `config_file` to `config_files`. If you are using this actor, please update your code. For example:
+
+   ```python
+   # Old
+   actor = ConfigManagingActor(config_file="config.toml")
+   # New
+   actor = ConfigManagingActor(config_files=["config.toml"])
+   ```
 
 ## New Features
 
-<!-- Here goes the main new features and examples or instructions on how to use them -->
+- The `ConfigManagingActor` can now take multiple configuration files as input, allowing to override default configurations with custom configurations.
 
 ## Bug Fixes
 

src/frequenz/sdk/config/_config_managing.py

@@ -7,6 +7,7 @@
 import pathlib
 import tomllib
 from collections import abc
+from collections.abc import Mapping, MutableMapping
 from datetime import timedelta
 from typing import Any, assert_never
 
@@ -19,21 +20,59 @@
 
 
 class ConfigManagingActor(Actor):
-    """An actor that monitors a TOML configuration file for updates.
-
-    When the file is updated, the new configuration is sent, as a [`dict`][], to the
-    `output` sender.
-
-    When the actor is started, if a configuration file already exists, then it will be
-    read and sent to the `output` sender before the actor starts monitoring the file
-    for updates. This way users can rely on the actor to do the initial configuration
-    reading too.
+    """An actor that monitors a TOML configuration files for updates.
+
+    When the actor is started the configuration files will be read and sent to the
+    output sender. Then the actor will start monitoring the files for updates. If any
+    file is updated, all the configuration files will be re-read and sent to the output
+    sender.
+
+    If no configuration file could be read, the actor will raise an exception.
+
+    The configuration files are read in the order of the paths, so the last path will
+    override the configuration set by the previous paths. Dict keys will be merged
+    recursively, but other objects (like lists) will be replaced by the value in the
+    last path.
+
+    Example:
+        If `config1.toml` contains:
+
+        ```toml
+        var1 = [1, 2]
+        var2 = 2
+        [section]
+        var3 = [1, 3]
+        ```
+
+        And `config2.toml` contains:
+
+        ```toml
+        var2 = "hello" # Can override with a different type too
+        var3 = 4
+        [section]
+        var3 = 5
+        var4 = 5
+        ```
+
+        Then the final configuration will be:
+
+        ```py
+        {
+            "var1": [1, 2],
+            "var2": "hello",
+            "var3": 4,
+            "section": {
+                "var3": 5,
+                "var4": 5,
+            },
+        }
+        ```
     """
 
     # pylint: disable-next=too-many-arguments
     def __init__(
         self,
-        config_path: pathlib.Path | str,
+        config_paths: abc.Sequence[pathlib.Path | str],
         output: Sender[abc.Mapping[str, Any]],
         event_types: abc.Set[EventType] = frozenset(EventType),
         *,
@@ -44,7 +83,11 @@ def __init__(
         """Initialize this instance.
 
         Args:
-            config_path: The path to the TOML file with the configuration.
+            config_paths: The paths to the TOML files with the configuration. Order
+                matters, as the configuration will be read and updated in the order
+                of the paths, so the last path will override the configuration set by
+                the previous paths. Dict keys will be merged recursively, but other
+                objects (like lists) will be replaced by the value in the last path.
             output: The sender to send the configuration to.
             event_types: The set of event types to monitor.
             name: The name of the actor. If `None`, `str(id(self))` will
@@ -54,11 +97,14 @@ def __init__(
                 polling is enabled.
         """
         super().__init__(name=name)
-        self._config_path: pathlib.Path = (
-            config_path
-            if isinstance(config_path, pathlib.Path)
-            else pathlib.Path(config_path)
-        )
+        self._config_paths: list[pathlib.Path] = [
+            (
+                config_path
+                if isinstance(config_path, pathlib.Path)
+                else pathlib.Path(config_path)
+            )
+            for config_path in config_paths
+        ]
         self._output: Sender[abc.Mapping[str, Any]] = output
         self._event_types: abc.Set[EventType] = event_types
         self._force_polling: bool = force_polling
@@ -73,12 +119,22 @@ def _read_config(self) -> abc.Mapping[str, Any]:
         Raises:
             ValueError: If config file cannot be read.
         """
-        try:
-            with self._config_path.open("rb") as toml_file:
-                return tomllib.load(toml_file)
-        except ValueError as err:
-            _logger.error("%s: Can't read config file, err: %s", self, err)
-            raise
+        error_count = 0
+        config: dict[str, Any] = {}
+
+        for config_path in self._config_paths:
+            try:
+                with config_path.open("rb") as toml_file:
+                    data = tomllib.load(toml_file)
+                    config = _recursive_update(config, data)
+            except ValueError as err:
+                _logger.error("%s: Can't read config file, err: %s", self, err)
+                error_count += 1
+
+        if error_count == len(self._config_paths):
+            raise ValueError(f"{self}: Can't read any of the config files")
+
+        return config
 
     async def send_config(self) -> None:
         """Send the configuration to the output sender."""
@@ -94,45 +150,72 @@ async def _run(self) -> None:
         """
         await self.send_config()
 
+        parent_paths = {p.parent for p in self._config_paths}
+
         # FileWatcher can't watch for non-existing files, so we need to watch for the
-        # parent directory instead just in case a configuration file doesn't exist yet
+        # parent directories instead just in case a configuration file doesn't exist yet
         # or it is deleted and recreated again.
         file_watcher = FileWatcher(
-            paths=[self._config_path.parent],
+            paths=list(parent_paths),
             event_types=self._event_types,
             force_polling=self._force_polling,
             polling_interval=self._polling_interval,
         )
 
         try:
             async for event in file_watcher:
-                # Since we are watching the whole parent directory, we need to make sure
-                # we only react to events related to the configuration file.
-                if not event.path.samefile(self._config_path):
+                # Since we are watching the whole parent directories, we need to make
+                # sure we only react to events related to the configuration files we
+                # are interested in.
+                if not any(event.path.samefile(p) for p in self._config_paths):
                     continue
 
                 match event.type:
                     case EventType.CREATE:
                         _logger.info(
                             "%s: The configuration file %s was created, sending new config...",
                             self,
-                            self._config_path,
+                            event.path,
                         )
                         await self.send_config()
                     case EventType.MODIFY:
                         _logger.info(
                             "%s: The configuration file %s was modified, sending update...",
                             self,
-                            self._config_path,
+                            event.path,
                         )
                         await self.send_config()
                     case EventType.DELETE:
                         _logger.info(
                             "%s: The configuration file %s was deleted, ignoring...",
                             self,
-                            self._config_path,
+                            event.path,
                         )
                     case _:
                         assert_never(event.type)
         finally:
             del file_watcher
+
+
+def _recursive_update(
+    target: dict[str, Any], overrides: Mapping[str, Any]
+) -> dict[str, Any]:
+    """Recursively updates dictionary d1 with values from dictionary d2.
+
+    Args:
+        target: The original dictionary to be updated.
+        overrides: The dictionary with updates.
+
+    Returns:
+        The updated dictionary.
+    """
+    for key, value in overrides.items():
+        if (
+            key in target
+            and isinstance(target[key], MutableMapping)
+            and isinstance(value, MutableMapping)
+        ):
+            _recursive_update(target[key], value)
+        else:
+            target[key] = value
+    return target