Merge branch 'main' into forman-21-append_step

Author: forman

CHANGES.md

@@ -2,6 +2,9 @@
 
 ### Enhancements
 
+* It is now possible to reference environment variables
+  in configuration files using the syntax `${ENV_VAR}`. [#36]
+
 * Added a demo Notebook `exmaples/zappend-demo.ipynb` and linked 
   it by a binder badge in README.md. [#47] 
 

docs/guide.md

@@ -50,6 +50,10 @@ zappend -t output/mycube.zarr -c config.yaml inputs/*.nc
 If multiple configuration files are passed, they will be merged into one by
 incrementally updating the first by subsequent ones. 
 
+!!! info "Environment Variables"
+    It is possible to include the values of environment variables in JSON or YAML 
+    configuration files using the syntax `${ENV_VAR}` or just `$ENV_VAR`.
+
 You can pass configuration settings to the `zappend` Python function with 
 the optional `config` keyword argument. Other keyword arguments are 
 interpreted as individual configuration settings and will be merged into the 

tests/test_config.py

@@ -3,6 +3,7 @@
 # https://opensource.org/licenses/MIT.
 
 import json
+import os
 import unittest
 
 import fsspec
@@ -83,6 +84,40 @@ def test_normalize_yaml_uri(self):
             f.write(yaml.dump(config))
         self.assertEqual(config, normalize_config(uri))
 
+    def test_interpolate_env_vars_json(self):
+        uri = "memory://config.json"
+        config = {
+            "slice_storage_options": {
+                "key": "${_TEST_S3_KEY}",
+                "secret": "$_TEST_S3_SECRET",
+            }
+        }
+        with fsspec.open(uri, "wt") as f:
+            f.write(json.dumps(config))
+        os.environ["_TEST_S3_KEY"] = "abc"
+        os.environ["_TEST_S3_SECRET"] = "123"
+        self.assertEqual(
+            {"slice_storage_options": {"key": "abc", "secret": "123"}},
+            normalize_config(uri),
+        )
+
+    def test_interpolate_env_vars_yaml(self):
+        uri = "memory://config.yaml"
+        config = {
+            "slice_storage_options": {
+                "key": "${_TEST_S3_KEY}",
+                "secret": "$_TEST_S3_SECRET",
+            }
+        }
+        with fsspec.open(uri, "w") as f:
+            f.write(yaml.dump(config))
+        os.environ["_TEST_S3_KEY"] = "abc"
+        os.environ["_TEST_S3_SECRET"] = "123"
+        self.assertEqual(
+            {"slice_storage_options": {"key": "abc", "secret": 123}},
+            normalize_config(uri),
+        )
+
     def test_normalize_file_obj(self):
         file_obj = FileObj("memory://config.yaml")
         config = {"version": 1, "zarr_version": 2}

zappend/config/config.py

@@ -2,10 +2,12 @@
 # Permissions are hereby granted under the terms of the MIT License:
 # https://opensource.org/licenses/MIT.
 
-from contextlib import contextmanager
+from typing import Any
+import contextlib
+import io
 import json
 import os.path
-from typing import Any
+import string
 
 import jsonschema
 import jsonschema.exceptions
@@ -91,10 +93,13 @@ def load_config(config_file: FileObj) -> dict[str, Any]:
     logger.info(f"Reading configuration {config_file.uri}")
     _, ext = os.path.splitext(config_file.path)
     with config_file.fs.open(config_file.path, "rt") as f:
-        if ext in yaml_extensions:
-            config = yaml.safe_load(f)
-        else:
-            config = json.load(f)
+        source = f.read()
+    source = string.Template(source).safe_substitute(os.environ)
+    stream = io.StringIO(source)
+    if ext in yaml_extensions:
+        config = yaml.safe_load(stream)
+    else:
+        config = json.load(stream)
     if not isinstance(config, dict):
         raise TypeError(
             f"Invalid configuration:" f" {config_file.uri}: object expected"
@@ -138,6 +143,6 @@ def _merge_values(value_1: Any, value_2: Any) -> Any:
     return value_2
 
 
-@contextmanager
+@contextlib.contextmanager
 def exclude_from_config(config: dict[str, Any], *keys: str) -> dict[str, Any]:
     yield {k: v for k, v in config.items() if k not in keys}

zappend/context.py

@@ -2,7 +2,6 @@
 # Permissions are hereby granted under the terms of the MIT License:
 # https://opensource.org/licenses/MIT.
 
-import inspect
 import tempfile
 from typing import Any, Dict