Merge pull request #192 from JonathonReinhart/191-expand-vars-in-volumes

Expand env variables in volume paths in configuration file

Author: JonathonReinhart

docs/configuration.rst

@@ -166,6 +166,19 @@ of volume options:
         hostpath: /host/foo
         options: ro,cached
 
+The keys used in volume mappings can contain environment variables **that are
+expanded in the host environment**. For example, this configuration would map
+the user's ``/home/username/.config/application1`` directory into the container
+at the same path.
+
+.. code-block:: yaml
+
+    volumes:
+      $TEST_HOME/.config/application1: $TEST_HOME/.config/application1
+
+Note that because variable expansion is now applied to all volume keys, if one
+desires to have a key with an explicit ``$`` character, it must be written as
+``$$``.
 
 
 .. _conf_aliases:

scuba/config.py

@@ -261,9 +261,22 @@ def _get_volumes(data):
 
     vols = {}
     for cpath, v in voldata.items():
+        cpath = _expand_path(cpath)
         vols[cpath] = ScubaVolume.from_dict(cpath, v)
     return vols
 
+def _expand_path(in_str):
+    try:
+        output = expand_env_vars(in_str)
+    except KeyError as ke:
+        # pylint: disable=raise-missing-from
+        raise ConfigError("Unset environment variable '{}' used in '{}'".format(ke.args[0], in_str))
+    except ValueError as ve:
+        raise ConfigError("Unable to expand string '{}' due to parsing "
+                "errors".format(in_str)) from ve
+
+    return output
+
 class ScubaVolume:
     def __init__(self, container_path, host_path=None, options=None):
         self.container_path = container_path
@@ -282,7 +295,7 @@ def from_dict(cls, cpath, node):
         if isinstance(node, str):
             return cls(
                 container_path = cpath,
-                host_path = node,
+                host_path = _expand_path(node),
                 )
 
         # Complex form
@@ -296,7 +309,7 @@ def from_dict(cls, cpath, node):
                 raise ConfigError("Volume {} must have a 'hostpath' subkey".format(cpath))
             return cls(
                 container_path = cpath,
-                host_path = hpath,
+                host_path = _expand_path(hpath),
                 options = _get_delimited_str_list(node, 'options', ','),
                 )
 

scuba/utils.py

@@ -1,6 +1,7 @@
 import errno
 import os
 from shlex import quote as shell_quote
+import string
 
 
 def shell_quote_cmd(cmdlist):
@@ -79,3 +80,10 @@ def get_umask():
 
 def writeln(f, line):
     f.write(line + '\n')
+
+def expand_env_vars(in_str):
+    """Expand environment variables in a string
+
+    Can raise `KeyError` if a variable is referenced but not defined, similar to
+    bash's nounset (set -u) option"""
+    return string.Template(in_str).substitute(os.environ)

tests/test_config.py

@@ -883,3 +883,76 @@ def test_alias_volumes_set(self):
         assert v.container_path == '/bar'
         assert v.host_path == '/host/bar'
         assert v.options == ['z', 'ro']
+
+    def test_volumes_with_env_vars_simple(self, monkeypatch):
+        '''volume definitions can contain environment variables'''
+        monkeypatch.setenv("TEST_VOL_PATH", "/bar/baz")
+        monkeypatch.setenv("TEST_VOL_PATH2", "/moo/doo")
+        with open('.scuba.yml', 'w') as f:
+            f.write(r'''
+                image: na
+                volumes:
+                  $TEST_VOL_PATH/foo: ${TEST_VOL_PATH2}/foo
+                ''')
+
+        config = scuba.config.load_config('.scuba.yml')
+        vols = config.volumes
+        assert len(vols) == 1
+
+        v = list(vols.values())[0]
+        assert isinstance(v, scuba.config.ScubaVolume)
+        assert v.container_path == '/bar/baz/foo'
+        assert v.host_path == '/moo/doo/foo'
+        assert v.options == []
+
+    def test_volumes_with_env_vars_complex(self, monkeypatch):
+        '''complex volume definitions can contain environment variables'''
+        monkeypatch.setenv("TEST_HOME", "/home/testuser")
+        monkeypatch.setenv("TEST_TMP", "/tmp")
+        monkeypatch.setenv("TEST_MAIL", "/var/spool/mail/testuser")
+
+        with open('.scuba.yml', 'w') as f:
+            f.write(r'''
+                image: na
+                volumes:
+                  $TEST_HOME/.config: ${TEST_HOME}/.config
+                  $TEST_TMP/:
+                    hostpath: $TEST_HOME/scuba/myproject/tmp
+                  /var/spool/mail/container:
+                    hostpath: $TEST_MAIL
+                    options: z,ro
+                ''')
+
+        config = scuba.config.load_config('.scuba.yml')
+        vols = config.volumes
+        assert len(vols) == 3
+
+        v = vols['/home/testuser/.config']
+        assert isinstance(v, scuba.config.ScubaVolume)
+        assert v.container_path == '/home/testuser/.config'
+        assert v.host_path == '/home/testuser/.config'
+        assert v.options == []
+
+        v = vols['/tmp/']
+        assert isinstance(v, scuba.config.ScubaVolume)
+        assert v.container_path == '/tmp/'
+        assert v.host_path == '/home/testuser/scuba/myproject/tmp'
+        assert v.options == []
+
+        v = vols['/var/spool/mail/container']
+        assert isinstance(v, scuba.config.ScubaVolume)
+        assert v.container_path == '/var/spool/mail/container'
+        assert v.host_path == "/var/spool/mail/testuser"
+        assert v.options == ['z', 'ro']
+
+    def test_volumes_with_invalid_env_vars(self, monkeypatch):
+        '''Volume definitions cannot include unset env vars'''
+        # Ensure that the entry does not exist in the environment
+        monkeypatch.delenv("TEST_VAR1", raising=False)
+        with open('.scuba.yml', 'w') as f:
+            f.write(r'''
+                image: na
+                volumes:
+                  $TEST_VAR1/foo: /host/foo
+                ''')
+        self._invalid_config('TEST_VAR1')

tests/test_main.py

@@ -1,6 +1,7 @@
 from .utils import *
 from unittest import mock
 import pytest
+import warnings
 
 import logging
 import os

tests/test_utils.py

@@ -128,3 +128,26 @@ def test_writeln():
         scuba.utils.writeln(s, 'hello')
         scuba.utils.writeln(s, 'goodbye')
         assert s.getvalue() == 'hello\ngoodbye\n'
+
+def test_expand_env_vars(monkeypatch):
+    monkeypatch.setenv("MY_VAR", "my favorite variable")
+    assert scuba.utils.expand_env_vars("This is $MY_VAR") == \
+            "This is my favorite variable"
+    assert scuba.utils.expand_env_vars("What is ${MY_VAR}?") == \
+            "What is my favorite variable?"
+
+def test_expand_missing_env_vars(monkeypatch):
+    monkeypatch.delenv("MY_VAR", raising=False)
+    # Verify that a KeyError is raised for unset env variables
+    with pytest.raises(KeyError) as kerr:
+        scuba.utils.expand_env_vars("Where is ${MY_VAR}?")
+    assert kerr.value.args[0] == "MY_VAR"
+
+
+def test_expand_env_vars_dollars():
+    # Verify that a ValueError is raised for bare, unescaped '$' characters
+    with pytest.raises(ValueError):
+        scuba.utils.expand_env_vars("Just a lonely $")
+
+    # Verify that it is possible to get '$' characters in an expanded string
+    assert scuba.utils.expand_env_vars(r"Just a lonely $$") == "Just a lonely $"