Merge pull request #403 from moremoban/dev

chfw · web-flow · commit df67d87eac65 · 2020-09-10T20:37:55.000+01:00
release 0.8.2
diff --git a/.github/workflows/moban-update.yml b/.github/workflows/moban-update.yml
@@ -14,7 +14,7 @@ jobs:
           python-version: '3.7'
       - name: check changes
         run: |
-          pip install moban gitfs2 pypifs moban-jinja2-github
+          pip install moban gitfs2 pypifs moban-jinja2-github moban-ansible
           make update
           git status
           git diff --exit-code
diff --git a/.moban.cd/changelog.yml b/.moban.cd/changelog.yml
@@ -1,6 +1,13 @@
 name: moban
 organisation: moremoban
 releases:
+  - changes:
+      - action: Fixed
+        details:
+          - Use any functions, any data structure of any python packages
+            as jinja2 filters, tests, globals
+    date: 04.09.2020
+    version: 0.8.2
   - changes:
       - action: Fixed
         details:
diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml
@@ -4,9 +4,9 @@ organisation: moremoban
 author: chfw
 contact: wangc_2011@hotmail.com
 license: MIT
-version: 0.8.1
-current_version: 0.8.1
-release: 0.8.1
+version: 0.8.2
+current_version: 0.8.2
+release: 0.8.2
 branch: master
 master: index
 command_line_interface: "moban"
diff --git a/.moban.d/moban_readme.jj2 b/.moban.d/moban_readme.jj2
@@ -40,15 +40,6 @@ package. `moban.plugins.jinja2.filters.github` is moved to moban-jinja2-github
 package Please install them for backward compatibility.
 
 
-From 2020 onwards, minimum requirement is Python 3.6
-
-
-For existing moban users, python 2 support has been dropped. Please stay with
-versions lower than 0.7.0 if you are still using python 2.
-
-
-
-
 Quick start
 ================================================================================
 
@@ -139,7 +130,32 @@ moban allows the injection of user preferred jinja2 extensions:
 
    $ moban -e jj2=jinja2_time.TimeExtension ...
 
-Can I write my own jinja2 test, filter and/or globals
+
+Well, can I nick some existing functions as filters, tests? Or create a global from another library?
+-----------------------------------------------------------------------------------------------------
+
+Sure, you can use the same '-e' syntax:
+
+.. code-block:: bash
+
+   $ moban -e jinja2=filter:module.path.filter_function \
+              jinja2=test:module.path.test_function \
+              jinja2=global:identifier=module.path.variable
+
+In this case, you would have to include the external library in your own requirements.txt
+
+Here is an example:
+
+{% raw %}
+.. code-block:: bash
+
+   $ moban -e jinja2=filter:moban.externals.file_system.url_join \
+     jinja2=test:moban.externals.file_system.exists \
+     jinja2=global:description=moban.constants.PROGRAM_DESCRIPTION \
+     -t "{{ 'a'|url_join('b')}} {{'b' is exists}}"
+{% endraw %}
+
+Can I write my own jinja2 test, filter and/or globals?
 -----------------------------------------------------------
 
 moban allows the freedom of craftsmanship. Please refer to the docs for more
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -1,6 +1,14 @@
 Change log
 ================================================================================
 
+0.8.2 - 04.09.2020
+--------------------------------------------------------------------------------
+
+**Fixed**
+
+#. Use any functions, any data structure of any python packages as jinja2
+   filters, tests, globals
+
 0.8.1 - 04.09.2020
 --------------------------------------------------------------------------------
 
diff --git a/README.rst b/README.rst
@@ -40,15 +40,6 @@ package. `moban.plugins.jinja2.filters.github` is moved to moban-jinja2-github
 package Please install them for backward compatibility.
 
 
-From 2020 onwards, minimum requirement is Python 3.6
-
-
-For existing moban users, python 2 support has been dropped. Please stay with
-versions lower than 0.7.0 if you are still using python 2.
-
-
-
-
 Quick start
 ================================================================================
 
@@ -137,7 +128,31 @@ moban allows the injection of user preferred jinja2 extensions:
 
    $ moban -e jj2=jinja2_time.TimeExtension ...
 
-Can I write my own jinja2 test, filter and/or globals
+
+Well, can I nick some existing functions as filters, tests? Or create a global from another library?
+-----------------------------------------------------------------------------------------------------
+
+Sure, you can use the same '-e' syntax:
+
+.. code-block:: bash
+
+   $ moban -e jinja2=filter:module.path.filter_function \
+              jinja2=test:module.path.test_function \
+              jinja2=global:identifier=module.path.variable
+
+In this case, you would have to include the external library in your own requirements.txt
+
+Here is an example:
+
+
+.. code-block:: bash
+
+   $ moban -e jinja2=filter:moban.externals.file_system.url_join \
+     jinja2=test:moban.externals.file_system.exists \
+     jinja2=global:description=moban.constants.PROGRAM_DESCRIPTION \
+     -t "{{ 'a'|url_join('b')}} {{'b' is exists}}"
+
+Can I write my own jinja2 test, filter and/or globals?
 -----------------------------------------------------------
 
 moban allows the freedom of craftsmanship. Please refer to the docs for more
diff --git a/docs/conf.py b/docs/conf.py
@@ -25,9 +25,9 @@
 copyright = '2017-2020 Onni Software Ltd.'
 author = 'chfw'
 # The short X.Y version
-version = '0.8.1'
+version = '0.8.2'
 # The full version, including alpha/beta/rc tags
-release = '0.8.1'
+release = '0.8.2'
 
 # -- General configuration ---------------------------------------------------
 
diff --git a/docs/index.rst b/docs/index.rst
@@ -59,6 +59,7 @@ Migration Notes
 
 .. toctree::
 
+   trouble-shooting-guide
    migration-note
 
    
diff --git a/docs/level-12-use-template-engine-extensions/.moban.yml b/docs/level-12-use-template-engine-extensions/.moban.yml
@@ -4,3 +4,6 @@ targets:
 extensions:
   jinja2:
     - jinja2.ext.with_
+    - filter:moban.externals.file_system.url_join
+    - test:moban.externals.file_system.exists
+    - global:description=moban.constants.PROGRAM_DESCRIPTION
diff --git a/docs/level-12-use-template-engine-extensions/README.rst b/docs/level-12-use-template-engine-extensions/README.rst
@@ -1,6 +1,12 @@
 Level 12: use template engine extensions
 ================================================================================
 
+There are three possible ways to provide extensions for your template engine.
+Let's take jinja2 as an example.
+
+1. Ready made extensions
+-----------------------------
+
 jinja2 comes with a lot of extensions. In order not to be the blocker in the
 middle, **extensions** is allowed in moban file to initialize jinja2 engine
 with desired extensions. Two extensions, expression-statement and loop-controls
@@ -21,9 +27,61 @@ For example::
 Please also note that the following extensions are included by default:
 `jinja2.ext.do`, `jinja2.ext.loopcontrols`
 
-.. note::
 
-   if you intend to use extensions for one off usage, please use '-e' cli option
+**Command line**
+
+if you intend to use extensions for one off usage, please use '-e' cli option.
+for example: `moban -e jinja2=your_custom_jinja2_extension <https://github.com/chfw/math-sheets/blob/master/reception/a_op_b_op_c/make.sh>`_
+
+ 
+2. Ad-hoc declaration
+-----------------------------
+
+Let's say you are fond of some existing functions, for example, ansible's combine
+filter. With moban, you can immediately include it for your template via the following
+syntax:
+
+.. code-block::
+
+   extensions:
+     jinja2:
+       - filter:module.path.filter_function
+       - test:module.path.test_function
+       - global:identifier=module.path.variable
+
+For example::
+
+   extensions:
+     jinja2:
+       - filter:ansible.plugins.filter.core.combine
+       - test:moban.externals.file_system.exists
+
+**Command line**
+
+.. code-block:: bash
+
+   $ moban -e jinja2=filter:module.path.filter_function jinja2=test:module.path.test_function jinja2=global:identifier=module.path.variable
+
+you can do this::
+
+   $ moban -e jinja2=filter:module.path.filter_function \
+              jinja2=test:module.path.test_function \
+              jinja2=global:identifier=module.path.variable
+
+
+3. Make your own extensions
+--------------------------------
+
+You can choose to write an extension for the template type of your choice.
+For example, you can write a reusable extension for jinja2. moban will be
+able to load it as it is.
+
+If you decide that you only want to write them for moban but for your own
+use, you can follow `Level 7: Custom jinja filters, tests and globals` and
+write your own. When you would like to make yours avaiable for all moban
+users, you can follow `moban-jinja2-github <https://github.com/moremoban/moban-jinja2-github>`_ and
+`moban-ansible <https://github.com/moremoban/moban-ansible>`_ 
+
 
 Evaluation
 --------------------------------------------------------------------------------
diff --git a/docs/level-12-use-template-engine-extensions/a.template b/docs/level-12-use-template-engine-extensions/a.template
@@ -1,3 +1,11 @@
 {% for _ in range(1,5) %}
 {{ hello }}
 {% endfor %}
+{% if 'b.template' is exists %}
+b.template exists
+{% endif %}
+{% if 'alien' is exists %}
+we are not the only intelligent life in the universe
+{% endif %}
+{{ 'a' | url_join('b')}}
+{{ description }}
diff --git a/docs/migration-notes.rst b/docs/migration-notes.rst
@@ -1,3 +1,20 @@
+Migrate to 0.8.x
+================================================================================
+
+In version 0.8.0, `moban.plugins.jinja2.tests.files` is moved to moban-ansible
+package. `moban.plugins.jinja2.filters.github` is moved to moban-jinja2-github
+package Please install them for backward compatibility.
+
+
+Migrate to 0.7.x
+================================================================================
+
+From 2020 onwards, minimum requirement is Python 3.6
+
+
+For existing moban users, python 2 support has been dropped. Please stay with
+versions lower than 0.7.0 if you are still using python 2.
+
 Migrate to 0.6.x
 ================================================================================
 
diff --git a/docs/trouble-shooting-guide.rst b/docs/trouble-shooting-guide.rst
@@ -0,0 +1,9 @@
+Trouble shooting guide
+==========================
+
+1. Why a file was not templated but copied instead?
+
+   It has been coded so that template engine can choose to pass on the template if it failed to handle. Moban will take over
+   and use default 'copy' action.
+
+   In order to find out what went wrong, you can use '-vvv' to enable all logs to assist you.
diff --git a/moban/_version.py b/moban/_version.py
@@ -1,2 +1,2 @@
-__version__ = "0.8.1"
+__version__ = "0.8.2"
 __author__ = "chfw"
diff --git a/moban/core/moban_factory.py b/moban/core/moban_factory.py
@@ -237,7 +237,8 @@ def _render_with_finding_template_first(self, template_file_index):
                         )
                         self.templated_count += 1
                     self.file_count += 1
-            except exceptions.PassOn:
+            except exceptions.PassOn as e:
+                LOG.info(e)
                 for (data_file, output) in data_output_pairs:
                     self.fall_out_targets.append(
                         TemplateTarget(
diff --git a/moban/main.py b/moban/main.py
@@ -8,6 +8,7 @@
     :license: MIT License, see LICENSE for more details
 
 """
+import re
 import sys
 import logging
 import argparse
@@ -299,8 +300,11 @@ def handle_custom_extensions(list_of_definitions):
     user_extensions = defaultdict(set)
     if list_of_definitions:
         for definition in list_of_definitions:
-            key, value = definition.split("=")
-            user_extensions[key].add(value)
+            result = re.match("(.*?)=(.*)", definition)
+            if result:
+                key, value = result.group(1), result.group(2)
+                user_extensions[key].add(value)
+
     ENGINES.register_extensions(user_extensions)
 
 
diff --git a/moban/plugins/jinja2/engine.py b/moban/plugins/jinja2/engine.py
diff --git a/setup.py b/setup.py
diff --git a/tests/integration_tests/test_command_line_options.py b/tests/integration_tests/test_command_line_options.py
diff --git a/tests/requirements.txt b/tests/requirements.txt
diff --git a/tests/test_docs.py b/tests/test_docs.py

Original file line number	Diff line number	Diff line change
`@@ -59,6 +59,7 @@ Migration Notes`
`59`	`59`
`60`	`60`	`.. toctree::`
`61`	`61`
	`62`	`+ trouble-shooting-guide`
`62`	`63`	`migration-note`
`63`	`64`
`64`	`65`