From d4a94882b2e1133db0c391818d662d168dcefa56 Mon Sep 17 00:00:00 2001 From: danic85 Date: Sun, 8 Jun 2025 16:17:27 +0100 Subject: [PATCH] Local override config files enabled --- .gitignore | 6 +- .mutagen.yml | 6 + .../overrides/vision_opencv.local.yml.example | 2 + docs/ModuleLoader.md | 120 ++++++++++++++++++ module_loader.py | 26 +++- 5 files changed, 157 insertions(+), 3 deletions(-) create mode 100644 .mutagen.yml create mode 100644 config/overrides/vision_opencv.local.yml.example create mode 100644 docs/ModuleLoader.md diff --git a/.gitignore b/.gitignore index e4d2250c..8eb7d71e 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,8 @@ myenv installers/i2s_mic_installed.flag installers/i2s_speech_recognition_installed.flag -speech.wav \ No newline at end of file +speech*.wav + +# ignore local configuration files +config/overrides/*.local.yml +config/local_overrides.yml \ No newline at end of file diff --git a/.mutagen.yml b/.mutagen.yml new file mode 100644 index 00000000..531d9cd9 --- /dev/null +++ b/.mutagen.yml @@ -0,0 +1,6 @@ +sync: + defaults: + ignore: + vcs: true + paths: + - config/overrides/*.local.yml diff --git a/config/overrides/vision_opencv.local.yml.example b/config/overrides/vision_opencv.local.yml.example new file mode 100644 index 00000000..5ad8bfa6 --- /dev/null +++ b/config/overrides/vision_opencv.local.yml.example @@ -0,0 +1,2 @@ +vision_opencv: + enabled: true diff --git a/docs/ModuleLoader.md b/docs/ModuleLoader.md new file mode 100644 index 00000000..60501fdd --- /dev/null +++ b/docs/ModuleLoader.md @@ -0,0 +1,120 @@ +# ModuleLoader Documentation + +## Overview + +The `ModuleLoader` class is responsible for dynamically loading and instantiating modules based on YAML configuration files. It supports both base (version-controlled) configuration and local instance-specific overrides, enabling flexible and environment-specific setups without modifying the main configuration files. + +## Key Features + +1. **Dynamic Module Loading**: Loads Python modules and creates instances based on configuration. +2. **YAML-Based Configuration**: Reads module definitions from YAML files in the `config/` directory. +3. **Local Overrides**: Supports local override YAML files (e.g., `config/overrides/*.local.yml`) to customize module settings per instance. +4. **Messaging Service Integration**: Can inject a messaging service into loaded modules. +5. **Multiple Instances**: Supports loading multiple instances of a module with different configurations. + +## How ModuleLoader Works + +- Scans the `config/` directory for `.yml` files describing modules. +- For each module, checks for a corresponding local override file in `config/overrides/` (e.g., `vision_imx500.local.yml`). +- Merges the local override into the base configuration, with the override taking precedence. +- Dynamically imports the specified Python class and instantiates it with the merged configuration. +- Returns a dictionary of module instances for use in your application. + +## Example: Base and Local Override YAML + +### Base Configuration (`config/vision_imx500.yml`) + +```yaml +vision_imx500: + enabled: true + path: modules.vision.imx500.vision.Vision + config: + preview: false + pin: 17 +``` + +### Local Override (`config/overrides/vision_imx500.local.yml`) + +```yaml +vision_imx500: + enabled: false + config: + preview: true + pin: 22 +``` + +> **Note:** Local override files have been added to `.gitignore` to avoid committing instance-specific settings. They have also been excluded from mutagen sync. + +## How to Use ModuleLoader + +### Loading Modules + +```python +# filepath: /home/dan/projects/modular-biped/main.py +from module_loader import ModuleLoader + +loader = ModuleLoader() +modules = loader.load_modules() + +# Access a loaded module instance +vision = modules['Vision'] +``` + +### Setting the Messaging Service + +```python +# filepath: /home/dan/projects/modular-biped/main.py +messaging_service = ... # Your messaging service instance +loader.set_messaging_service(modules, messaging_service) +``` + +## Creating Local Override YAML Files + +1. **Create an override file** in `config/overrides/` with the same base name as the module config, but with `.local.yml` extension. +2. **Specify only the keys you want to override** (e.g., `enabled`, `config` values). +3. **Do not commit override files** to version control. + +**Example Directory Structure:** +``` +config/ + vision_imx500.yml + ... + overrides/ + vision_imx500.local.yml +.gitignore +``` + +**Example `.gitignore` Entry:** +``` +config/overrides/*.local.yml +``` + +## Deep Merge Behavior + +When both base and override YAML files are present, the loader recursively merges the override into the base. This means only the specified keys in the override file will replace or extend the base configuration. + +**Example Merge Result:** + +Base: +```yaml +config: + preview: false + pin: 17 + foo: bar +``` +Override: +```yaml +config: + preview: true +``` +Result: +```yaml +config: + preview: true + pin: 17 + foo: bar +``` + +## Conclusion + +The `ModuleLoader` class provides a robust and flexible way to manage module configuration and instantiation in your project. By leveraging local override YAML files, you can easily customize behavior for different environments or hardware setups without modifying the main configuration files or risking merge conflicts in version control. diff --git a/module_loader.py b/module_loader.py index 2d21f5e8..9bb06a6c 100644 --- a/module_loader.py +++ b/module_loader.py @@ -3,8 +3,17 @@ import importlib.util from pubsub import pub +def deep_merge(a, b): + """Recursively merge dict b into dict a.""" + for k, v in b.items(): + if isinstance(v, dict) and k in a and isinstance(a[k], dict): + a[k] = deep_merge(a[k], v) + else: + a[k] = v + return a + class ModuleLoader: - def __init__(self, config_folder='config'): + def __init__(self, config_folder='config', override_folder='config/overrides'): """ ModuleLoader class :param config_folder: folder containing the module configuration files @@ -27,16 +36,29 @@ def __init__(self, config_folder='config'): translator_inst = modules['Translator'] """ self.config_folder = config_folder + self.override_folder = override_folder self.modules = self.load_yaml_files() def load_yaml_files(self): - """Load and parse YAML files from the config folder.""" + """Load and parse YAML files from the config folder, merging with local overrides if present.""" config_files = [os.path.join(self.config_folder, f) for f in os.listdir(self.config_folder) if f.endswith('.yml')] loaded_modules = [] for file_path in config_files: with open(file_path, 'r') as stream: try: config = yaml.safe_load(stream) + # Try to load override file + base_filename = os.path.basename(file_path) + override_path = os.path.join(self.override_folder, base_filename.replace('.yml', '.local.yml')) + if os.path.exists(override_path): + with open(override_path, 'r') as o_stream: + override = yaml.safe_load(o_stream) + # Merge override into base config + for module_name, module_config in override.items(): + if module_name in config: + config[module_name] = deep_merge(config[module_name], module_config) + else: + config[module_name] = module_config for module_name, module_config in config.items(): if module_config.get('enabled', False): loaded_modules.append(module_config)