Add CLI --set flag for generic config overrides and improve config API

- Add --set flag to override any config parameter with dot notation
- Remove --detect-anomaly flag (use --set model.detect_anomaly=true instead)
- Make config utility functions public (remove underscore prefix)
- Add comprehensive type hints to config.py
- Switch CLI to use load_config() for consistent config handling
- Move config imports to top-level (PEP 8 compliance)
- Add CLI override documentation to config_loader.rst
- Update help examples to show --set usage

Author: francois-drielsma

docs/source/config_loader.rst

@@ -184,6 +184,42 @@ Usage in Python
    print(cfg['io']['loader']['batch_size'])  # 8
    print(cfg['model']['modules']['uresnet']['depth'])  # 5
 
+Command-Line Overrides
+----------------------
+
+When using the SPINE CLI, you can override any configuration parameter using the ``--set`` flag with dot notation:
+
+.. code-block:: bash
+
+   # Override a single parameter
+   spine -c config.yaml --set io.loader.batch_size=8
+
+   # Override multiple parameters
+   spine -c config.yaml \
+     --set base.iterations=1000 \
+     --set io.loader.batch_size=16 \
+     --set io.loader.dataset.file_keys=[file1.root,file2.root]
+
+   # Mix with other CLI options
+   spine -c config.yaml \
+     --source /data/input.root \
+     --output /data/output.h5 \
+     --set model.weight_path=/weights/model.ckpt
+
+The ``--set`` flag accepts any valid YAML value:
+
+- **Strings**: ``--set model.name=my_model``
+- **Numbers**: ``--set base.iterations=1000`` or ``--set base.learning_rate=0.001``
+- **Booleans**: ``--set io.loader.shuffle=true``
+- **Lists**: ``--set io.loader.dataset.file_keys=[file1.root,file2.root]``
+- **Nested paths**: ``--set io.loader.dataset.schema.data.num_features=8``
+
+This is particularly useful for:
+
+- **Hyperparameter sweeps**: Quickly test different values without editing config files
+- **Production runs**: Override paths and settings for different environments
+- **Debugging**: Enable/disable features or adjust batch sizes on the fly
+
 Benefits
 --------
 

src/spine/bin/cli.py

@@ -6,7 +6,7 @@
 import pathlib
 import sys
 
-import yaml
+from spine.utils.config import load_config, parse_value, set_nested_value
 
 
 def main(
@@ -16,10 +16,10 @@ def main(
     output,
     n,
     nskip,
-    detect_anomaly,
     log_dir,
     weight_prefix,
     weight_path,
+    config_overrides,
 ):
     """Main driver for training/validation/inference/analysis.
 
@@ -41,15 +41,15 @@ def main(
         Number of iterations to run
     nskip : int
         Number of iterations to skip
-    detect_anomaly : bool
-        Whether to turn on anomaly detection in torch
     log_dir : str
         Path to the directory for storing the training log
     weight_prefix : str
         Path to the directory for storing the training weights
     weight_path : str
         Path string a weight file or pattern for multiple weight files to load
         the model weights
+    config_overrides : List[str]
+        List of config overrides in the form "key.path=value"
     """
     # Try to find configuration file using the absolute path or under
     # the 'config' directory relative to the current working directory
@@ -60,9 +60,8 @@ def main(
     if not os.path.isfile(cfg_file):
         raise FileNotFoundError(f"Configuration not found: {config}")
 
-    # Load the configuration file
-    with open(cfg_file, "r", encoding="utf-8") as cfg_yaml:
-        cfg = yaml.safe_load(cfg_yaml)
+    # Load the configuration file using the advanced loader
+    cfg = load_config(cfg_file)
 
     # If there is no base block, build one
     if "base" not in cfg:
@@ -117,12 +116,24 @@ def main(
     if weight_path is not None:
         cfg["model"]["weight_path"] = weight_path
 
-    # Turn on PyTorch anomaly detection, if requested
-    if detect_anomaly:
-        assert (
-            "model" in cfg
-        ), "There is no model to detect anomalies for, add `model` block."
-        cfg["model"]["detect_anomaly"] = detect_anomaly
+    # Apply any generic config overrides from --set arguments
+    if config_overrides:
+        for override in config_overrides:
+            if "=" not in override:
+                raise ValueError(
+                    f"Invalid --set format: '{override}'. "
+                    f"Expected format: 'key.path=value'"
+                )
+
+            key_path, value_str = override.split("=", 1)
+            key_path = key_path.strip()
+            value_str = value_str.strip()
+
+            # Parse the value (handles strings, numbers, booleans, lists, etc.)
+            value = parse_value(value_str)
+
+            # Set the nested value
+            cfg = set_nested_value(cfg, key_path, value)
 
     # For actual training/inference, we need the main functionality
     from spine.main import run
@@ -138,10 +149,13 @@ def cli():
         formatter_class=argparse.RawDescriptionHelpFormatter,
         epilog="""
 Examples:
-  spine --version         Show version information
-  spine --info           Show system and dependency info
-  spine config.cfg       Run ML training/inference with config file
-  spine --help           Show this help message
+  spine --version                                Show version information
+  spine --info                                  Show system and dependency info
+  spine -c config.cfg                           Run ML training/inference with config file
+  spine -c config.cfg --set io.loader.batch_size=8    Override config parameters
+  spine -c config.cfg --set base.iterations=1000 --set io.loader.batch_size=16
+  spine -c config.cfg --set model.detect_anomaly=true Debug PyTorch issues
+  spine --help                                  Show this help message
 
 For ML training/inference functionality, ensure PyTorch is installed:
   pip install spine-ml[model]
@@ -187,12 +201,6 @@ def cli():
 
     parser.add_argument("--nskip", type=int, help="Number of iterations to skip")
 
-    parser.add_argument(
-        "--detect-anomaly",
-        action="store_true",
-        help="Whether to turn on anomaly detection in torch",
-    )
-
     parser.add_argument(
         "--log-dir", help="Path to the directory for storing the training log"
     )
@@ -206,6 +214,16 @@ def cli():
         help="Path string a weight file or pattern for multiple weight files to load model weights",
     )
 
+    parser.add_argument(
+        "--set",
+        action="append",
+        dest="config_overrides",
+        metavar="KEY=VALUE",
+        help="Override any config parameter using dot notation "
+        "(e.g., --set io.loader.batch_size=8). "
+        "Can be used multiple times for multiple overrides.",
+    )
+
     args = parser.parse_args()
 
     # If no arguments provided and no config, show help
@@ -228,10 +246,10 @@ def cli():
         output=args.output,
         n=args.iterations,
         nskip=args.nskip,
-        detect_anomaly=args.detect_anomaly,
         log_dir=args.log_dir,
         weight_prefix=args.weight_prefix,
         weight_path=args.weight_path,
+        config_overrides=args.config_overrides or [],
     )