Merge pull request #104 from matthias-wende-frequenz/load_paths

Refactor config file arguments of MicrogridData and MicrogridConfig classes

Author: matthias-wende-frequenz

RELEASE_NOTES.md

@@ -11,8 +11,10 @@
 ## New Features
 
 - Added consistent logger setup across all modules for structured logging and improved observability. Example notebooks updated to demonstrate logger usage.
+- The signature for passing config files MicrogridConfig.load_config() has been changed to accept a path a list of paths and a directory containing the config files.
+- `MicrogridData` class needs to be initialized with a `MicrogridConfig` object instead of a path to config file(s).
 
 ## Bug Fixes
 
 - Fixed a bug in the notification `Scheduler` where tasks could overrun the configured duration due to imprecise sleep and stop logic. The scheduler now correctly tracks elapsed time, respects task execution duration, and stops reliably after the intended interval.
-- Fixed an issue where `EmailNotification` did not properly initialise its scheduler. Also fixed an example in the docstring.
+- Fixed an issue where `EmailNotification` did not properly initialise its scheduler. Also fixed an example in the docstring.

src/frequenz/data/microgrid/component_data.py

@@ -24,30 +24,20 @@ def __init__(
         server_url: str,
         auth_key: str,
         sign_secret: str,
-        microgrid_config_path: str | list[str],
+        microgrid_configs: dict[str, MicrogridConfig] | None = None,
     ) -> None:
         """Initialize microgrid data.
 
         Args:
             server_url: URL of the reporting service.
             auth_key: Authentication key to the service.
             sign_secret: Secret for signing requests.
-            microgrid_config_path: Path(s) to the config file with microgrid components.
-
-        Raises:
-            ValueError: If no microgrid config path is provided.
+            microgrid_configs: MicrogridConfig dict mapping microgrid IDs to MicrogridConfigs.
         """
+        self._microgrid_configs = microgrid_configs
         self._client = ReportingApiClient(
             server_url=server_url, auth_key=auth_key, sign_secret=sign_secret
         )
-        paths = (
-            [microgrid_config_path]
-            if isinstance(microgrid_config_path, str)
-            else microgrid_config_path
-        )
-        if len(paths) < 1:
-            raise ValueError("At least one microgrid config path must be provided")
-        self._microgrid_configs = MicrogridConfig.load_configs(*paths)
 
     @property
     def microgrid_ids(self) -> list[str]:
@@ -98,6 +88,8 @@ async def metric_data(  # pylint: disable=too-many-arguments
             ctype: mcfg.formula(ctype, metric.upper()) for ctype in component_types
         }
 
+        logging.debug("Formulas: %s", formulas)
+
         metric_enum = Metric[metric.upper()]
         data = [
             sample

src/frequenz/data/microgrid/config.py

@@ -3,11 +3,15 @@
 
 """Configuration for microgrids."""
 
+import logging
 import re
 import tomllib
 from dataclasses import dataclass
+from pathlib import Path
 from typing import Any, Literal, cast, get_args
 
+_logger = logging.getLogger(__name__)
+
 ComponentType = Literal["grid", "pv", "battery", "consumption", "chp", "ev"]
 """Valid component types."""
 
@@ -286,22 +290,71 @@ def formula(self, component_type: str, metric: str) -> str:
         return formula
 
     @staticmethod
-    def load_configs(*paths: str) -> dict[str, "MicrogridConfig"]:
+    def load_configs(
+        microgrid_config_files: str | Path | list[str | Path] | None = None,
+        microgrid_config_dir: str | Path | None = None,
+    ) -> dict[str, "MicrogridConfig"]:
         """Load multiple microgrid configurations from a file.
 
         Configs for a single microgrid are expected to be in a single file.
         Later files with the same microgrid ID will overwrite the previous configs.
 
         Args:
-            *paths: Path(es) to the config file(s).
+            microgrid_config_files: Path to a single microgrid config file or list of paths.
+            microgrid_config_dir: Directory containing multiple microgrid config files.
 
         Returns:
             Dictionary of single microgrid formula configs with microgrid IDs as keys.
+
+        Raises:
+            ValueError: If no config files or dir is provided, or if no config files are found.
         """
-        microgrid_configs = {}
-        for config_path in paths:
-            with open(config_path, "rb") as f:
+        if microgrid_config_files is None and microgrid_config_dir is None:
+            raise ValueError(
+                "No microgrid config path or directory provided. "
+                "Please provide at least one."
+            )
+
+        config_files: list[Path] = []
+
+        if microgrid_config_files:
+            if isinstance(microgrid_config_files, str):
+                config_files = [Path(microgrid_config_files)]
+            elif isinstance(microgrid_config_files, Path):
+                config_files = [microgrid_config_files]
+            elif isinstance(microgrid_config_files, list):
+                config_files = [Path(f) for f in microgrid_config_files]
+
+        if microgrid_config_dir:
+            if Path(microgrid_config_dir).is_dir():
+                config_files += list(Path(microgrid_config_dir).glob("*.toml"))
+            else:
+                raise ValueError(
+                    f"Microgrid config directory {microgrid_config_dir} "
+                    "is not a directory"
+                )
+
+        if len(config_files) == 0:
+            raise ValueError(
+                "No microgrid config files found. "
+                "Please provide at least one valid config file."
+            )
+
+        microgrid_configs: dict[str, "MicrogridConfig"] = {}
+
+        for config_path in config_files:
+            if not config_path.is_file():
+                _logger.warning("Config path %s is not a file, skipping.", config_path)
+                continue
+
+            with config_path.open("rb") as f:
                 cfg_dict = tomllib.load(f)
                 for microgrid_id, mcfg in cfg_dict.items():
+                    _logger.debug(
+                        "Loading microgrid config for ID %s from %s",
+                        microgrid_id,
+                        config_path,
+                    )
                     microgrid_configs[microgrid_id] = MicrogridConfig(mcfg)
+
         return microgrid_configs

tests/test_config.py

@@ -3,6 +3,7 @@
 
 """Tests for the frequenz.lib.notebooks.config module."""
 
+from pathlib import Path
 from typing import Any, cast
 
 import pytest
@@ -122,12 +123,14 @@ def test_load_configs(mocker: MockerFixture) -> None:
     1.ctype.battery.component = [301, 302, 303, 304, 305, 306]
     1.pv.PV1.peak_power = 5000
     1.pv.PV1.rated_power = 4500
-    1.pv.PV2peak_power = 8000
-    1.pv.PV2rated_power = 7000
+    1.pv.PV2.peak_power = 8000
+    1.pv.PV2.rated_power = 7000
     1.battery.BAT1.capacity = 10000
     """
-    mocker.patch("builtins.open", mocker.mock_open(read_data=toml_data.encode("utf-8")))
-    configs = MicrogridConfig.load_configs("mock_path.toml")
+    mock_file = mocker.mock_open(read_data=toml_data.encode("utf-8"))
+    mocker.patch("pathlib.Path.open", mock_file)
+    mocker.patch("pathlib.Path.is_file", mocker.Mock(return_value=True))
+    configs = MicrogridConfig.load_configs(Path("mock_path.toml"))
 
     assert "1" in configs
     assert configs["1"].meta.name == "Test Grid"