Merge pull request #26 from con2/index

Add Index and Group tags

Author: japsu

README.md

@@ -78,21 +78,23 @@ Python 3.5+ required. Python 2 is not and will not be supported.
 | `!Exists` | JSONPath expression | `!Exists foo` | Returns `true` if the JSONPath expression returns one or more matches, `false` otherwise. |
 | `!Filter` | `test`, `over` | See `tests/test_cond.py` | Takes in a list and only returns elements that pass a predicate. |
 | `!Format` | Format string | `!Format "{foo} {bar!d}"` | Interpolate strings using [Python format strings](https://docs.python.org/3/library/string.html#formatstrings). <br> JSONPath supported in variable lookup (eg. `{people[0].first_name}` will do the right thing). <br> **NOTE:** When the format string starts with `{`, you need to quote it in order to avoid being interpreted as a YAML object. |
+| `!Group` | Accepts the same arguments as `!Loop`, except `template` is optional (default identity), plus the following: <br> `by`: (required) An expression used to determine the key for the current value <br> `result_as`: (optional, string) When evaluating `by`, the enriched `template` is available under this name. | TBD | Makes a dict out of a list. Keys are determined by `by`. Items with the same key are grouped in a list. |
 | `!If` | `test`, `then`, `else` | See `tests/test_cond.py` | Returns one of two values based on a condition. |
 | `!Include` | Path to a template to include | `!Include ../foo.yml` | Renders the requested template at this location. Both absolute and relative paths work. |
 | `!IncludeBase64` | Path to a binary file | `!IncludeBase64 ../foo.pdf` | Loads the given binary file and returns the contents encoded as Base64. |
 | `!IncludeBinary` | Path to a binary file | `!IncludeBinary ../foo.pdf` | Loads the given binary file and returns the contents as bytes.  This is practically only useful for hashing. |
 | `!IncludeText` | Path to an UTF-8 text file | `!IncludeText ../foo.toml` | Loads the given UTF-8 text file and returns the contents as a string. |
+| `!Index` | Accepts the same arguments as `!Loop`, except `template` is optional (default identity), plus the following: <br> `by`: (required) An expression used to determine the key for the current value <br> `result_as`: (optional, string) When evaluating `by`, the enriched `template` is available under this name. <br> `duplicates`: (optional, default `error`) `error`, `warn(ing)` or `ignore` duplicate values. | TBD | Makes a dict out of a list. Keys are determined by `by`. |
 | `!Join` | `items`: (required) A list of items to be joined together. <br> `separator`: (optional, default space) The separator to place between the items. <br> **OR** <br> a list of items to be joined together with a space as the separator. | `!Join [foo, bar]` <br> `!Join { items: [foo, bar], separator: ', ' }` | Joins a list of items together with a separator. The result is always a string. |
 | `!Lookup` | JSONPath expression | `!Lookup people[0].first_name` | Performs a JSONPath lookup returning the first match. If there is no match, an error is raised. |
 | `!LookupAll` | JSONPath expression | `!LookupAll people[*].first_name` | Performs a JSONPath lookup returning all matches as a list. If no matches are found, the empty list `[]` is returned. |
-| `!Loop` | `over`: (required) The data to iterate over (a literal list or dict, or !Var) <br> `as`: (optional, default `item`) The variable name given to the current value <br> `index_as`: (optional) The variable name given to the loop index. If over is a list, this is a numeric index starting from `0`. If over is a dict, this is the dict key. <br> `index_start`: (optional, default `0`) First index, for eg. 1-based indexing. <br> `template`: (required) The template to process for each iteration of the loop. | See `examples/loop/`. | Loops over a list or dict and renders a template for each iteration. The output is always a list. |
-| `!MD5` | Data to hash | `!MD5 'Törkylempijävongahdus' | Hashes the given data using the MD5 algorithm. If the data is not binary, it is converted to UTF-8 bytes. |
+| `!Loop` | `over`: (required) The data to iterate over (a literal list or dict, or !Var) <br> `as`: (optional, default `item`) The variable name given to the current value <br> `index_as`: (optional) The variable name given to the loop index. If over is a list, this is a numeric index starting from `0`. If over is a dict, this is the dict key. <br> `index_start`: (optional, default `0`) First index, for eg. 1-based indexing. <br> `previous_as`: (optional) The variable name given to the previous value. On the first iteration of the loop, the previous value is `null`. _Added in 0.2.0_ <br> `template`: (required) The template to process for each iteration of the loop. | See `examples/loop/`. | Loops over a list or dict and renders a template for each iteration. The output is always a list. |
+| `!MD5` | Data to hash | `!MD5 'some data to hash'` | Hashes the given data using the MD5 algorithm. If the data is not binary, it is converted to UTF-8 bytes. |
 | `!Merge` | A list of dicts | `!Merge [{a: 5}, {b: 6}]` | Merges objects. For overlapping keys the last one takes precedence. |
 | `!Not` | a value | `!Not !Var foo` | Logically negates the given value (in Python semantics). |
 | `!Op` | `a`, `op`, `b` | See `tests/test_cond.py` | Performs binary operators. Especially useful with `!If` to implement greater-than etc. |
-| `!SHA1` | Data to hash | `!SHA1 'Törkylempijävongahdus' | Hashes the given data using the SHA1 algorithm. If the data is not binary, it is converted to UTF-8 bytes. |
-| `!SHA256` | Data to hash | `!SHA256 'Törkylempijävongahdus' | Hashes the given data using the SHA256 algorithm. If the data is not binary, it is converted to UTF-8 bytes. |
+| `!SHA1` | Data to hash | `!SHA1 'some data to hash'` | Hashes the given data using the SHA1 algorithm. If the data is not binary, it is converted to UTF-8 bytes. |
+| `!SHA256` | Data to hash | `!SHA256 'some data to hash'` | Hashes the given data using the SHA256 algorithm. If the data is not binary, it is converted to UTF-8 bytes. |
 | `!URLEncode` | A string to encode <br> **OR** <br> `url`: The URL to combine query parameters into <br> `query`: An object of query string parameters to add. | `!URLEncode "foo+bar"` <br> `!URLEncode { url: "https://example.com/", query: { foo: bar } }` | Encodes strings for safe inclusion in a URL, or combines query string parameters into a URL. |
 | `!Var` | Variable name | `!Var image_name` | Replaced with the value of the variable. |
 | `!Void` | Anything or nothing | `foo: !Void` | The dict key, list item or YAML document that resolves to `!Void` is removed from the output. |

emrichen/tags/__init__.py

@@ -14,6 +14,7 @@
 from .hash import MD5, SHA1, SHA256
 from .if_ import If
 from .include import Include, IncludeBase64, IncludeBinary, IncludeText
+from .index import Group, Index
 from .join import Join
 from .lookup import Lookup, LookupAll
 from .loop import Loop
@@ -37,11 +38,13 @@
     'Exists',
     'Filter',
     'Format',
+    'Group',
     'If',
     'Include',
     'IncludeBase64',
     'IncludeBinary',
     'IncludeText',
+    'Index',
     'Join',
     'Lookup',
     'LookupAll',

emrichen/tags/index.py

@@ -0,0 +1,83 @@
+from collections import OrderedDict
+from sys import stderr
+
+from .loop import Loop
+from .var import Var
+
+
+class _BaseIndex(Loop):
+    value_types = (dict,)
+    output_factory = OrderedDict
+
+    def __init__(self, data):
+        if 'template' not in data:
+            as_ = data.get('as', 'item')
+            data = dict(data, template=Var(as_))
+
+        super(_BaseIndex, self).__init__(data)
+
+
+class Index(_BaseIndex):
+    """
+    arguments: |
+        Accepts the same arguments as `!Loop`, except `template` is optional (default identity), plus the following:
+        `by`: (required) An expression used to determine the key for the current value
+        `result_as`: (optional, string) When evaluating `by`, the enriched `template` is available under this name.
+        `duplicates`: (optional, default `error`) `error`, `warn(ing)` or `ignore` duplicate values.
+
+    example: TBD
+    description: Makes a dict out of a list. Keys are determined by `by`.
+    """
+    def __init__(self, data):
+        if 'template' not in data:
+            as_ = data.get('as', 'item')
+            data = dict(data, template=Var(as_))
+
+        super(Index, self).__init__(data)
+
+    def process_item(self, context, output, value, result):
+        from ..context import Context
+
+        by = self.data['by']
+        result_as = self.data.get('result_as')
+
+        if result_as:
+            context = Context(context, {result_as: result})
+
+        key = context.enrich(by)
+
+        if key in output:
+            # Duplicate key
+            action = self.data.get('duplicates', 'error')
+            message = '{self}: Duplicate key {key!r}'.format(self=self, key=key)
+            if action == 'warn':
+                # TODO logger
+                stderr.write(message + '\n')
+            elif action == 'error':
+                raise ValueError(message)
+
+        output[key] = result
+
+
+class Group(_BaseIndex):
+    """
+    arguments: |
+        Accepts the same arguments as `!Loop`, except `template` is optional (default identity), plus the following:
+        `by`: (required) An expression used to determine the key for the current value
+        `result_as`: (optional, string) When evaluating `by`, the enriched `template` is available under this name.
+
+    example: TBD
+    description: Makes a dict out of a list. Keys are determined by `by`. Items with the same key are grouped in a list.
+    """
+    def process_item(self, context, output, value, result):
+        from ..context import Context
+
+        by = self.data['by']
+        result_as = self.data.get('result_as')
+
+        if result_as:
+            context = Context(context, {result_as: result})
+
+        key = context.enrich(by)
+        group = output.setdefault(key, [])
+        group.append(result)

emrichen/tags/loop.py

@@ -40,12 +40,13 @@ class Loop(BaseTag):
         `as`: (optional, default `item`) The variable name given to the current value
         `index_as`: (optional) The variable name given to the loop index. If over is a list, this is a numeric index starting from `0`. If over is a dict, this is the dict key.
         `index_start`: (optional, default `0`) First index, for eg. 1-based indexing.
+        `previous_as`: (optional) The variable name given to the previous value. On the first iteration of the loop, the previous value is `null`. _Added in 0.2.0_
         `template`: (required) The template to process for each iteration of the loop.
     example: See `examples/loop/`.
     description: Loops over a list or dict and renders a template for each iteration. The output is always a list.
-
     """
     value_types = (dict,)
+    output_factory = list
 
     def enrich(self, context):
         from ..context import Context
@@ -54,22 +55,37 @@ def enrich(self, context):
         index_as = str(self.data.get('index_as') or '')
         compact = bool(self.data.get('compact'))
         index_start = self.data.get('index_start')
+        previous_as = str(self.data.get('previous_as') or '')
 
         template = self.data.get('template')
         if template is None:
             raise ValueError('{self}: missing template'.format(self=self))
 
-        output = []
+        output = self.output_factory()
         iterable, _ = self.get_iterable(context, index_start)
+        previous_value = None
         for index, value in iterable:
             subcontext = {as_: value}
             if index_as:
                 subcontext[index_as] = index
-            value = Context(context, subcontext).enrich(template)
-            if compact and not value:
+            if previous_as:
+                subcontext[previous_as] = previous_value
+            subcontext = Context(context, subcontext)
+
+            result = subcontext.enrich(template)
+            previous_value = value
+
+            if compact and not result:
                 continue
-            output.append(value)
+
+            self.process_item(subcontext, output, value, result)
         return output
 
     def get_iterable(self, context, index_start):
         return get_iterable(self, self.data.get('over'), context, index_start)
+
+    def process_item(self, context, output, value, result):
+        '''
+        Used by Loop subclasses to do things every iteration.
+        '''
+        output.append(result)

setup.py

@@ -9,7 +9,7 @@
 
 setup(
     name='emrichen',
-    version='0.1.1',
+    version='0.2.0',
     description='Template engine for YAML & JSON',
     long_description=long_description,
     long_description_content_type='text/markdown',

tests/test_group.py

@@ -0,0 +1,17 @@
+from emrichen import Template
+
+
+def test_group():
+    assert Template.parse('''
+!Group
+  over:
+    - name: manifold
+      score: 7.8
+    - name: John
+      score: 9.9
+    - name: John
+      score: 9.8
+  as: flavour
+  by: !Lookup flavour.name
+  template: !Lookup flavour.score
+    ''').enrich({}) == [{'manifold': [7.8], 'John': [9.9, 9.8]}]

tests/test_index.py

@@ -0,0 +1,74 @@
+import pytest
+
+from emrichen import Template
+
+
+def test_index():
+    assert Template.parse('''
+!Index
+  over:
+    - name: manifold
+      score: 7.8
+    - name: John
+      score: 9.9
+    - name: John
+      score: 9.8
+  as: flavour
+  by: !Lookup flavour.name
+  duplicates: ignore
+  template: !Lookup flavour.score
+    ''').enrich({}) == [{'manifold': 7.8, 'John': 9.8}]
+
+
+def test_index_without_template():
+    assert Template.parse('''
+!Index
+  over:
+    - name: manifold
+      score: 7.8
+    - name: John
+      score: 9.9
+    - name: John
+      score: 9.8
+  as: flavour
+  by: !Lookup flavour.name
+  duplicates: ignore
+    ''').enrich({}) == [{'manifold': {'name': 'manifold', 'score': 7.8}, 'John': {'name': 'John', 'score': 9.8}}]
+
+
+def test_index_result_as():
+    assert Template.parse('''
+!Index
+  over:
+    - name: manifold
+      score: 7.8
+    - name: John
+      score: 9.9
+    - name: John
+      score: 9.8
+  as: flavour
+  template:
+    NAME: !Lookup flavour.name
+    SCORE: !Lookup flavour.score
+  result_as: result
+  by: !Lookup result.NAME
+  duplicates: ignore
+    ''').enrich({}) == [{'manifold': {'NAME': 'manifold', 'SCORE': 7.8}, 'John': {'NAME': 'John', 'SCORE': 9.8}}]
+
+
+def test_index_duplicates_error():
+    with pytest.raises(ValueError):
+        assert Template.parse('''
+!Index
+  over:
+    - name: manifold
+      score: 7.8
+    - name: John
+      score: 9.9
+    - name: John
+      score: 9.8
+  as: flavour
+  by: !Lookup flavour.name
+  duplicates: error
+  template: !Lookup flavour.score
+        ''').enrich({})

tests/test_loop.py

@@ -90,3 +90,13 @@ def test_no_index_start_with_dict():
         template.render({})
 
     assert 'index_start with dict' in str(nope.value)
+
+
+def test_previous_as():
+    assert Template.parse('''
+!Loop
+  over: [1, 2, 3]
+  as: item
+  previous_as: previous_item
+  template: [!Var previous_item, !Var item]
+''').enrich({}) == [[[None, 1], [1, 2], [2,3]]]

update_tags_init.py

@@ -8,26 +8,11 @@
 from collections import defaultdict
 
 
-class ClassDefWalker(ast.NodeVisitor):
+class Universe:
     def __init__(self):
+        self.cls_modules = {}
         self.cls_bases = defaultdict(set)
 
-    def _stringify_id_node(self, node):
-        if isinstance(node, ast.Name):
-            return node.id
-        elif isinstance(node, ast.Attribute):
-            return f'{self._stringify_id_node(node.value)}.{self._stringify_id_node(node.attr)}'
-        elif isinstance(node, str):
-            return node
-        else:
-            warnings.warn(f'Dont know how to stringify {node}')
-            return '<none>'
-
-    def visit_ClassDef(self, classdef):
-        for base in classdef.bases:
-            base_name = self._stringify_id_node(base)
-            self.cls_bases[classdef.name].add(base_name)
-
     def get_all_bases(self, name):
         bases = set()
         open = list(self.cls_bases[name])
@@ -38,28 +23,53 @@ def get_all_bases(self, name):
                 bases.add(name)
                 open.extend(list(self.cls_bases[name]))
 
-    def get_classes_deriving_from(self, base_name):
+    def get_classes_deriving_from(self, base_names):
         for name in list(self.cls_bases.keys()):
             for base in self.get_all_bases(name):
-                if base == base_name:
+                if base in base_names:
                     yield name
                     break
 
 
+class ClassDefWalker(ast.NodeVisitor):
+    def __init__(self, module, universe):
+        self.module = module
+        self.universe = universe
+
+    def _stringify_id_node(self, node):
+        if isinstance(node, ast.Name):
+            return node.id
+        elif isinstance(node, ast.Attribute):
+            return f'{self._stringify_id_node(node.value)}.{self._stringify_id_node(node.attr)}'
+        elif isinstance(node, str):
+            return node
+        else:
+            warnings.warn(f'Dont know how to stringify {node}')
+            return '<none>'
+
+    def visit_ClassDef(self, classdef):
+        for base in classdef.bases:
+            base_name = self._stringify_id_node(base)
+            self.universe.cls_bases[classdef.name].add(base_name)
+            self.universe.cls_modules[classdef.name] = self.module
+
+
 def find_tags_in_modules():
     tags_per_module = defaultdict(set)
-
+    universe = Universe()
     for filepath in glob.glob('emrichen/tags/*.py'):
         modname = os.path.splitext(os.path.basename(filepath))[0]
         if modname.startswith('_'):
             continue
         with open(filepath) as infp:
             tree = ast.parse(infp.read(), filepath)
-            cdw = ClassDefWalker()
+            cdw = ClassDefWalker(module=modname, universe=universe)
             cdw.visit(tree)
-            for tag in cdw.get_classes_deriving_from('BaseTag'):
-                if not tag.startswith('_'):
-                    tags_per_module[modname].add(tag)
+
+    for tag in universe.get_classes_deriving_from(('BaseTag',)):
+        if not tag.startswith('_'):
+            tags_per_module[universe.cls_modules[tag]].add(tag)
+
     return tags_per_module