Re-work hab Formatter class to handle non-defined env vars

- The parse method now converts !e to literal_text when not expanding or if the env var is not set.
- The get_field method now takes care of resolving existing env var's to strings.
- Add the docs directory to .gitignore

Author: MHendricks

.gitignore

@@ -11,6 +11,7 @@ __pycache__/
 build/
 coverage.xml
 dist/
+docs/
 htmlcov/
 venv/
 .venv/

README.md

@@ -880,7 +880,8 @@ with these extra features:
 * `{ANYTHING!e}`: `!e` is a special conversion flag for Environment variables. This will
 be replaced with the correct shell environment variable. For bash it becomes `$ANYTHING`,
 in power shell `$env:ANYTHING`, and in command prompt `%ANYTHING%`. `ANYTHING` is the name
-of the environment variable.
+of the environment variable. In some cases hab will expand these into the current
+environment variable if set, if the env var is not set the shell env variable will be used.
 
 #### Hab specific variables
 

hab/formatter.py

@@ -7,18 +7,27 @@
 class Formatter(string.Formatter):
     """A extended string formatter class to parse habitat configurations.
 
-    Adds support for the "!e" conversion flag. This will fill in the key as a properly
-    formatted environment variable specifier. For example ''{PATH!e}'' will be converted
-    to ``$PATH`` for the sh language, and ``%env:PATH`` for the ps language. You can
-    convert "!e" to "!s" by setting expand to True. This simulates `os.path.expandvars`.
+    Adds support for the ``!e`` `conversion field`_. This will fill in the key as a
+    properly formatted environment variable specifier. For example ``{PATH!e}`` will
+    be converted to ``$PATH`` for the sh language, and ``%env:PATH`` for the ps language.
+
+    By setting expand to True the ``!e`` conversion field will be filled in with the
+    environment variable for that key if the env variable is set. Otherwise it will
+    fall back the environment variable specifier.
 
     This also converts ``{;}`` to the language specific path separator for environment
     variables. On linux this is ``:`` on windows(even in bash) this is ``;``.
 
-    You need to specify the desired language when initializing this class. This can be
-    one of the supported language keys in ''Formatter.shell_formats'', or you can pass
-    a file extension supported by `Formatter.language_from_ext`. If you pass None, it
-    will preserve the formatting markers so it can be converted later.
+    Parameters:
+        language: Specify the shell language to use when converting ``!e``. See
+            :py:const:`Formatter.shell_formats` and :py:meth:`Formatter.language_from_ext`.
+            for supported values. If you pass None, it will preserve the
+            formatting markers so it can be converted later.
+        expand(bool, optional): Should the ``!e`` conversion insert the shell
+            environment variable specifier or the value of the env var?
+
+    .. _conversion field:
+       https://docs.python.org/3/library/string.html#grammar-token-format-string-conversion
     """
 
     shell_formats = {
@@ -50,28 +59,35 @@ class Formatter(string.Formatter):
             ";": "{;}",
         },
     }
+    """Information on how to generate shell specific environment variable references
+    and the character to use for pathsep. For each shell ``env_var`` is a format
+    string that accepts the env var name. ``;`` is the path separator to use.
+    """
 
     def __init__(self, language, expand=False):
         super().__init__()
         self.language = self.language_from_ext(language)
-        self.current_field_name = None
         self.expand = expand
 
-    def convert_field(self, value, conversion):
-        if conversion == "e":
-            # Expand the env var to the real string value simulating `os.path.expandvars`
-            if self.expand:
-                return super().convert_field(value, "s")
+    def get_field(self, field_name, args, kwargs):
+        """Returns the object to be inserted for the given field_name.
 
-            # Otherwise insert the correct shell script env var reference
-            return self.shell_formats[self.language]["env_var"].format(
-                self.current_field_name
-            )
+        If kwargs doesn't contain ``field_name`` but ``field_name`` is in the
+        environment variables, the stored value is returned. This also returns
+        the pathsep for the ``;`` field_name. Otherwise works the same as
+        the standard `string.Formatter`_.
 
-        return super().convert_field(value, conversion)
+        .. _`string.Formatter`:
+           https://docs.python.org/3/library/string.html#string.Formatter.get_field
+        """
+        # If a field_name was not provided, use the value stored in os.environ
+        if field_name not in kwargs and field_name in os.environ:
+            return os.getenv(field_name), field_name
+        # Process the pathsep character
+        if field_name == ";":
+            value = self.shell_formats[self.language][";"]
+            return value, field_name
 
-    def get_field(self, field_name, args, kwargs):
-        self.current_field_name = field_name
         ret = super().get_field(field_name, args, kwargs)
         return ret
 
@@ -97,16 +113,19 @@ def language_from_ext(cls, ext):
                 return "sh"
         return ext
 
-    def merge_kwargs(self, kwargs):
-        """Merge the provided kwargs on top of the current language's shell_formats
-        dict. This makes it so the default format options are added by default, but
-        still allows us to override them if required
-        """
-        ret = dict(self.shell_formats[self.language], **kwargs)
-        ret = dict(os.environ, **ret)
-        return ret
-
-    def vformat(self, format_string, args, kwargs):
-        kwargs = self.merge_kwargs(kwargs)
-        ret = super().vformat(format_string, args, kwargs)
-        return ret
+    def parse(self, txt):
+        for literal_text, field_name, format_spec, conversion in super().parse(txt):
+            # Non-hab specific operation, just use the super value unchanged
+            if conversion != "e":
+                yield (literal_text, field_name, format_spec, conversion)
+                continue
+
+            elif self.expand and field_name in os.environ:
+                # Expand the env var to the env var value. Later `get_field`
+                # will update kwargs with the existing env var value
+                yield (literal_text, field_name, format_spec, "s")
+                continue
+
+            # Convert this !e conversion to the shell specific env var specifier
+            value = self.shell_formats[self.language]["env_var"].format(field_name)
+            yield (literal_text + value, None, None, None)

tests/test_formatter.py

@@ -1,5 +1,3 @@
-import os
-
 import pytest
 
 from hab import utils
@@ -8,28 +6,39 @@
 
 
 @pytest.mark.parametrize(
-    "language,shell,pathsep",
+    "language,expanded,not_expanded",
     (
-        ("sh", "-$PATH-", "-:-"),
+        ("sh", "_V_a-var_:_$INVALID_", "_V_$VALID_:_$INVALID_"),
         # Bash formatting is different on windows for env vars
-        ("shwin", "-$PATH-", "-:-"),
-        ("ps", "-$env:PATH-", "-;-"),
-        ("batch", "-%PATH%-", "-;-"),
-        (None, "-{PATH!e}-", "-{;}-"),
+        ("shwin", "_V_a-var_:_$INVALID_", "_V_$VALID_:_$INVALID_"),
+        ("ps", "_V_a-var_;_$env:INVALID_", "_V_$env:VALID_;_$env:INVALID_"),
+        ("batch", "_V_a-var_;_%INVALID%_", "_V_%VALID%_;_%INVALID%_"),
+        (None, "_V_a-var_{;}_{INVALID!e}_", "_V_{VALID!e}_{;}_{INVALID!e}_"),
     ),
 )
-def test_e_format(language, shell, pathsep):
-    """Check that "{VAR_NAME!e}" is properly formatted."""
-    path = os.environ["PATH"]
+def test_env_format(language, expanded, not_expanded, monkeypatch):
+    """Check that the custom Formatter class works as expected.
+
+    This check tests:
+        - You can still use pass kwargs like a normal Formatter.
+        - ``!e`` is converted for set env vars if ``expand==True``.
+          ``VALID`` becomes ``a-var``.
+        - ``!e`` uses the shell env specifier for set env vars if ``expand==False``.
+          (``VALID`` becomes ``$VALID`` for bash.)
+        - ``!e`` uses the shell env specifier for unset env variables.
+          (``INVALID`` becomes ``$INVALID`` for bash.)
+        - ``{;}`` gets converted to the shell's ``:`` or ``;``.
+    """
+    monkeypatch.setenv("VALID", "a-var")
+    monkeypatch.delenv("INVALID", raising=False)
+
+    fmt = "_{regular_var}_{VALID!e}_{;}_{INVALID!e}_"
 
     # Check that "!e" is converted to the correct shell specific specifier.
-    assert Formatter(language).format("-{PATH!e}-") == shell
+    assert Formatter(language).format(fmt, regular_var="V") == not_expanded
 
     # Check that "!e" uses the env var value if `expand=True` not the shell specifier.
-    assert Formatter(language, expand=True).format("-{PATH!e}-") == f"-{path}-"
-
-    # Check that the pathsep variable `{;}` is converted to the correct value
-    assert Formatter(language).format("-{;}-") == pathsep
+    assert Formatter(language, expand=True).format(fmt, regular_var="V") == expanded
 
 
 def test_language_from_ext(monkeypatch):