Merge pull request #51 from bcdev/forman-36-env_vars_in_config

Allow for env vars in config files

Author: forman

CHANGES.md

@@ -2,6 +2,9 @@
 
 ### Enhancements
 
+* It is now possible to reference environment variables
+  in configuration files using the syntax `${ENV_VAR}`. [#36]
+
 * Added a demo Notebook `exmaples/zappend-demo.ipynb` and linked 
   it by a binder badge in README.md. [#47] 
 

tests/test_config.py

@@ -3,6 +3,7 @@
 # https://opensource.org/licenses/MIT.
 
 import json
+import os
 import unittest
 
 import fsspec
@@ -83,6 +84,40 @@ def test_normalize_yaml_uri(self):
             f.write(yaml.dump(config))
         self.assertEqual(config, normalize_config(uri))
 
+    def test_interpolate_env_vars_json(self):
+        uri = "memory://config.json"
+        config = {
+            "slice_storage_options": {
+                "key": "${_TEST_S3_KEY}",
+                "secret": "$_TEST_S3_SECRET",
+            }
+        }
+        with fsspec.open(uri, "wt") as f:
+            f.write(json.dumps(config))
+        os.environ["_TEST_S3_KEY"] = "abc"
+        os.environ["_TEST_S3_SECRET"] = "123"
+        self.assertEqual(
+            {"slice_storage_options": {"key": "abc", "secret": "123"}},
+            normalize_config(uri),
+        )
+
+    def test_interpolate_env_vars_yaml(self):
+        uri = "memory://config.yaml"
+        config = {
+            "slice_storage_options": {
+                "key": "${_TEST_S3_KEY}",
+                "secret": "$_TEST_S3_SECRET",
+            }
+        }
+        with fsspec.open(uri, "w") as f:
+            f.write(yaml.dump(config))
+        os.environ["_TEST_S3_KEY"] = "abc"
+        os.environ["_TEST_S3_SECRET"] = "123"
+        self.assertEqual(
+            {"slice_storage_options": {"key": "abc", "secret": 123}},
+            normalize_config(uri),
+        )
+
     def test_normalize_file_obj(self):
         file_obj = FileObj("memory://config.yaml")
         config = {"version": 1, "zarr_version": 2}
@@ -233,6 +268,8 @@ def test_exclude_from_config(self):
         with exclude_from_config({"a": 1, "b": 2}, "b", "a") as config:
             self.assertEqual({}, config)
 
+
+class ConfigSchemaTest(unittest.TestCase):
     def test_get_config_schema(self):
         schema = get_config_schema()
         self.assertIn("properties", schema)

zappend/config/config.py

@@ -2,10 +2,12 @@
 # Permissions are hereby granted under the terms of the MIT License:
 # https://opensource.org/licenses/MIT.
 
-from contextlib import contextmanager
+from typing import Any
+import contextlib
+import io
 import json
 import os.path
-from typing import Any
+import string
 
 import jsonschema
 import jsonschema.exceptions
@@ -91,10 +93,13 @@ def load_config(config_file: FileObj) -> dict[str, Any]:
     logger.info(f"Reading configuration {config_file.uri}")
     _, ext = os.path.splitext(config_file.path)
     with config_file.fs.open(config_file.path, "rt") as f:
-        if ext in yaml_extensions:
-            config = yaml.safe_load(f)
-        else:
-            config = json.load(f)
+        source = f.read()
+    source = string.Template(source).safe_substitute(os.environ)
+    stream = io.StringIO(source)
+    if ext in yaml_extensions:
+        config = yaml.safe_load(stream)
+    else:
+        config = json.load(stream)
     if not isinstance(config, dict):
         raise TypeError(
             f"Invalid configuration:" f" {config_file.uri}: object expected"
@@ -138,6 +143,6 @@ def _merge_values(value_1: Any, value_2: Any) -> Any:
     return value_2
 
 
-@contextmanager
+@contextlib.contextmanager
 def exclude_from_config(config: dict[str, Any], *keys: str) -> dict[str, Any]:
     yield {k: v for k, v in config.items() if k not in keys}