Initial logic

Alan Christie · Alan Christie · commit 5d5e8ff5c89a · 2022-01-11T12:09:32.000Z
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -0,0 +1,47 @@
+---
+name: build
+
+# -----------------
+# Control variables (GitHub Secrets)
+# -----------------
+#
+# At the GitHub 'organisation' or 'project' level you must have the following
+# GitHub 'Repository Secrets' defined (i.e. via 'Settings -> Secrets'): -
+#
+# (none)
+#
+# -----------
+# Environment (GitHub Environments)
+# -----------
+#
+# Environment         (none)
+
+on:
+- push
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:
+        - '3.10'
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install requirements
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r build-requirements.txt
+        pip install -r package-requirements.txt
+    - name: Test
+      run: |
+        pytest --cov=decoder
+        pyroma .
+    - name: Build
+      run: |
+        python setup.py bdist_wheel
diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml
@@ -0,0 +1,52 @@
+---
+name: publish
+
+# Actions for any tag.
+
+# -----------------
+# Control variables (GitHub Secrets)
+# -----------------
+#
+# At the GitHub 'organisation' or 'project' level you must have the following
+# GitHub 'Repository Secrets' defined (i.e. via 'Settings -> Secrets'): -
+#
+# PYPI_USERNAME
+# PYPI_TOKEN
+#
+# -----------
+# Environment (GitHub Environments)
+# -----------
+#
+# Environment         (none)
+
+on:
+  push:
+    tags:
+    - '**'
+
+jobs:
+  build-and-publish:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Inject slug/short variables
+      uses: rlespinasse/github-slug-action@v3.x
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.10'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r build-requirements.txt
+        pip install -r package-requirements.txt
+    - name: Build
+      run: |
+        pyroma .
+        python setup.py bdist_wheel
+    - name: Publish
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        user: ${{ secrets.PYPI_USERNAME }}
+        password: ${{ secrets.PYPI_TOKEN }}
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,9 @@
+.idea
+.pytest_cache
+.coverage
+*.egg-info
+**/__pycache__
+venv
+build
+dist
+
diff --git a/LICENSE b/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Informatics Matters Ltd
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -0,0 +1 @@
+include LICENSE
diff --git a/README.md b/README.md
diff --git a/README.rst b/README.rst
@@ -0,0 +1,23 @@
+Informatics Matters Data Manager Job Decoder
+============================================
+
+.. image:: https://badge.fury.io/py/im-dm-job-decoder.svg
+   :target: https://badge.fury.io/py/im-dm-job-decoder
+   :alt: PyPI package (latest)
+
+Installation (Python)
+=====================
+
+The Job decoder is published on `PyPI`_ and can be installed from
+there::
+
+    pip install im-dm-job-decoder
+
+.. _PyPI: https://pypi.org/project/im-dm-job-decoder
+
+Get in touch
+============
+
+- Report bugs, suggest features or view the source code `on GitHub`_.
+
+.. _on GitHub: https://github.com/informaticsmatters/data-manager-job-decoder
diff --git a/build-requirements.txt b/build-requirements.txt
@@ -0,0 +1,3 @@
+pyroma == 3.2
+pytest == 6.2.5
+pytest-cov == 3.0.0
diff --git a/decoder/__init__.py b/decoder/__init__.py
diff --git a/decoder/decode_jinja2_3_0.py b/decoder/decode_jinja2_3_0.py
@@ -0,0 +1,51 @@
+"""A decoder for jinja2 3.0.x formatted strings.
+"""
+from typing import Dict, Optional, Set, Tuple
+
+import jinja2
+from jinja2.exceptions import TemplateSyntaxError
+from jinja2.meta import find_undeclared_variables
+
+
+def decode(template_text: str,
+           variable_map: Optional[Dict[str, str]],
+           subject: str) -> Tuple[str, bool]:
+    """Decodes text expected to conform to Jinja2 (v3.0.x)
+    """
+    assert template_text
+    assert subject
+
+    # Make a template from the text
+    env: jinja2.Environment = jinja2.\
+        Environment(undefined=jinja2.DebugUndefined)
+    try:
+        template: jinja2.Template = env.from_string(template_text)
+    except TemplateSyntaxError as ex:
+        msg: str = f'TemplateSyntaxError with {subject}: {ex}'
+        return msg, False
+
+    # Render (this works even if there are variables in the rendered text
+    # The rendered text, when stripped of whitespace, must not be empty.
+    rendered_text = template.render(variable_map).strip()
+    if len(rendered_text) == 0:
+        msg = f'Rendered text for {subject} is blank'
+        return msg, False
+
+    # Check if rendering was done correctly.
+    # It's a little odd with Jinja2 - as undefined variables are not
+    # considered errors. See https://stackoverflow.com/a/55699590 in the
+    # StackOverflow topic: -
+    #   How to get ALL undefined variables from a Jinja2 template?
+    abstract_syntax_tree = env.parse(rendered_text)
+    undefined: Set[str] = find_undeclared_variables(abstract_syntax_tree)
+    if undefined:
+        # We
+        msg = f'Undefined template variables for {subject}:'
+        for variable in undefined:
+            msg += f' {variable},'
+        # Return the message, stripping the last comma from it
+        return msg[:-1], False
+
+    # OK if we get here.
+    # Just return the rendered text
+    return rendered_text, True
diff --git a/decoder/decoder.py b/decoder/decoder.py
@@ -0,0 +1,47 @@
+"""A module to decode text strings based on an encoding.
+
+This is typically used by the Data Manager's API instance methods
+when launching applications (and Jobs) where text requires decoding,
+given a 'template' string and a 'dictionary' of parameters and values.
+"""
+import enum
+from typing import Dict, Optional, Tuple
+
+# The decoding engine implementations.
+# The modules are expected to be called 'decode_<TextEncoding.lower()>'
+from . import decode_jinja2_3_0
+
+
+class TextEncoding(enum.Enum):
+    """A general text encoding format, used initially for Job text fields.
+    """
+    JINJA2_3_0 = 1      # Encoding that complies with Jinja2 v3.0.x
+
+
+def decode(template_text: str,
+           variable_map: Optional[Dict[str, str]],
+           subject: str,
+           template_engine: TextEncoding) -> Tuple[str, bool]:
+    """Given some text and a 'variable map' (a dictionary of keys and values)
+    this returns the decoded text (using the named engine) as a string
+    and a boolean set to True. On failure the boolean is False and the returned
+    string is an error message.
+
+    The 'subject' is a symbolic reference to the text
+    used for error reporting so the client understands what text is
+    in error. i.e. if the text is for a 'command'
+    the subject might be 'command'.
+    """
+    assert template_text
+    assert subject
+    assert template_engine
+
+    # If there are no variables just return the template text
+    if variable_map is None:
+        return template_text, True
+
+    if template_engine.name.lower() == 'jinja2_3_0':
+        return decode_jinja2_3_0.decode(template_text, variable_map, subject)
+
+    # Unsupported engine if we get here!
+    return f'Unsupported template engine: {template_engine}', False
diff --git a/package-requirements.txt b/package-requirements.txt
@@ -0,0 +1,9 @@
+# These are the requirements to package the solution, assuming the testing
+# stage has successfully completed.
+#
+# pip install -r build-requirements.txt
+# pip install -r requirements.txt
+# pip install -r package-requirements.txt
+
+wheel == 0.37.1
+twine == 3.7.1
diff --git a/pytest.ini b/pytest.ini
@@ -0,0 +1,5 @@
+[pytest]
+testpaths =
+    tests
+markers =
+    unit: Marks a test for execution during unit testing
diff --git a/requirements.txt b/requirements.txt
@@ -0,0 +1 @@
+jinja2 == 3.0.3
diff --git a/setup.cfg b/setup.cfg
@@ -0,0 +1,5 @@
+[bdist_wheel]
+python-tag = py3
+
+[metadata]
+description_file = README.rst
diff --git a/setup.py b/setup.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Setup module for the Data Manager Job Decoder module
+#
+# January 2022
+
+from setuptools import setup
+import os
+
+# Pull in the essential run-time requirements
+with open('requirements.txt') as file:
+    requirements = file.read().splitlines()
+
+
+# Use the README.rst as the long description.
+def get_long_description():
+    return open('README.rst').read()
+
+
+setup(
+
+    name='im-dm-job-decoder',
+    version=os.environ.get('GITHUB_REF_SLUG', '1.0.0'),
+    author='Alan Christie',
+    author_email='achristie@informaticsmatters.com',
+    url='https://github.com/informaticsmatters/data-manager-job-decoder',
+    license='MIT',
+    description='Job decoding logic',
+    long_description=get_long_description(),
+    keywords='jinja2 decoder',
+    platforms=['any'],
+
+    # Our modules to package
+    packages=['decoder'],
+
+    # Project classification:
+    # https://pypi.python.org/pypi?%3Aaction=list_classifiers
+    classifiers=[
+        'Development Status :: 5 - Production/Stable',
+        'Environment :: Other Environment',
+        'Intended Audience :: Developers',
+        'License :: OSI Approved :: MIT License',
+        'Programming Language :: Python :: 3.10',
+        'Topic :: Software Development :: Libraries :: Python Modules',
+        'Operating System :: POSIX :: Linux',
+    ],
+
+    install_requires=requirements,
+
+    zip_safe=False,
+
+)
diff --git a/tests/__init__.py b/tests/__init__.py
diff --git a/tests/test_decode_jinja2_3_0.py b/tests/test_decode_jinja2_3_0.py
@@ -0,0 +1,54 @@
+# Tests for the decoder package.
+import pytest
+pytestmark = pytest.mark.unit
+
+from typing import Dict
+
+from decoder import decode_jinja2_3_0
+
+
+def test_basic_decode():
+    # Arrange
+    text: str = 'foo={{ foo }}, bar={{ bar }}, baz={{ baz }}'
+    field_map: Dict = {'foo': 1, 'bar': 2, 'baz': 3}
+    expected_text: str = 'foo=1, bar=2, baz=3'
+
+    # Act
+    rendered, success = decode_jinja2_3_0.decode(text, field_map, 'text')
+
+    # Assert
+    assert success
+    assert rendered == expected_text
+
+
+def test_decode_with_missing_variables():
+    # Arrange
+    text: str = 'foo={{ foo }}, bar={{ bar }}, baz={{ baz }}'
+    field_map: Dict = {'foo': 1}
+    expected_text_prefix: str = 'Undefined template variables for text:'
+
+    # Act
+    rendered, success = decode_jinja2_3_0.decode(text, field_map, 'text')
+
+    # Assert
+    assert not success
+    # The order of missing variables is not deterministic.
+    # They change - so we just need to make suer they're there.
+    assert rendered.startswith(expected_text_prefix)
+    assert 'bar' in rendered
+    assert 'baz' in rendered
+
+
+def test_decode_with_template_error():
+    # Arrange
+    text: str = 'foo={{ foo doo }}, bar={{ bar }}, baz={{ baz }}'
+    field_map: Dict = {'foo': 1}
+    expected_text: str = "TemplateSyntaxError with text:" \
+                         " expected token 'end of print statement', got 'doo'"
+
+    # Act
+    rendered, success = decode_jinja2_3_0.decode(text, field_map, 'text')
+
+    # Assert
+    assert not success
+    assert rendered == expected_text
diff --git a/tests/test_decoder.py b/tests/test_decoder.py

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+pyroma == 3.2`
	`2`	`+pytest == 6.2.5`
	`3`	`+pytest-cov == 3.0.0`