Refactor handling of Jinja2 templates and add filters

.vale/styles/spelling-exceptions.txt

@@ -85,6 +85,7 @@ namespace
 namespaces
 Nautobot
 Netbox
+Netutils
 Newsfragment
 Nornir
 npm

changelog/+5660f1dc.added.md

@@ -0,0 +1 @@
+Refactored management of Jinja2 templating to allow for filters within Infrahub. Builtin filters as well as those from Netutils are available.

docs/_templates/sdk_template_reference.j2

@@ -0,0 +1,27 @@
+---
+title: Python SDK Templating
+---
+Filters can be used when defining [computed attributes](https://docs.infrahub.app/guides/computed-attributes) or [Jinja2 Transforms](https://docs.infrahub.app/guides/jinja2-transform) within Infrahub.
+
+## Builtin Jinja2 filters
+
+The following filters are those that are [shipped with Jinja2](https://jinja.palletsprojects.com/en/stable/templates/#list-of-builtin-filters) and enabled within Infrahub. The trusted column indicates if the filter is allowed for use with Infrahub's computed attributes when the server is configured in strict mode.
+
+<!-- vale off -->
+| Name | Trusted |
+|----------|----------|
+{% for filter in builtin %}
+| {{ filter.name }} | {% if filter.trusted %}✅{% else %}❌{% endif %} |
+{% endfor %}
+<!-- vale on -->
+
+## Netutils filters
+
+The following Jinja2 filters from <a href="https://netutils.readthedocs.io">Netutils</a> are included within Infrahub.
+<!-- vale off -->
+| Name | Trusted |
+|----------|----------|
+{% for filter in netutils %}
+| {{ filter.name }} | {% if filter.trusted %}✅{% else %}❌{% endif %} |
+{% endfor %}
+<!-- vale on -->

docs/docs/python-sdk/reference/templating.mdx

@@ -0,0 +1,153 @@
+---
+title: Python SDK Templating
+---
+Filters can be used when defining [computed attributes](https://docs.infrahub.app/guides/computed-attributes) or [Jinja2 Transforms](https://docs.infrahub.app/guides/jinja2-transform) within Infrahub.
+
+## Builtin Jinja2 filters
+
+The following filters are those that are [shipped with Jinja2](https://jinja.palletsprojects.com/en/stable/templates/#list-of-builtin-filters) and enabled within Infrahub. The trusted column indicates if the filter is allowed for use with Infrahub's computed attributes when the server is configured in strict mode.
+
+<!-- vale off -->
+| Name | Trusted |
+|----------|----------|
+| abs | ✅ |
+| attr | ❌ |
+| batch | ❌ |
+| capitalize | ✅ |
+| center | ✅ |
+| count | ✅ |
+| d | ✅ |
+| default | ✅ |
+| dictsort | ❌ |
+| e | ✅ |
+| escape | ✅ |
+| filesizeformat | ✅ |
+| first | ✅ |
+| float | ✅ |
+| forceescape | ✅ |
+| format | ✅ |
+| groupby | ❌ |
+| indent | ✅ |
+| int | ✅ |
+| items | ❌ |
+| join | ✅ |
+| last | ✅ |
+| length | ✅ |
+| list | ✅ |
+| lower | ✅ |
+| map | ❌ |
+| max | ✅ |
+| min | ✅ |
+| pprint | ❌ |
+| random | ❌ |
+| reject | ❌ |
+| rejectattr | ❌ |
+| replace | ✅ |
+| reverse | ✅ |
+| round | ✅ |
+| safe | ❌ |
+| select | ❌ |
+| selectattr | ❌ |
+| slice | ✅ |
+| sort | ❌ |
+| string | ✅ |
+| striptags | ✅ |
+| sum | ✅ |
+| title | ✅ |
+| tojson | ❌ |
+| trim | ✅ |
+| truncate | ✅ |
+| unique | ❌ |
+| upper | ✅ |
+| urlencode | ✅ |
+| urlize | ❌ |
+| wordcount | ✅ |
+| wordwrap | ✅ |
+| xmlattr | ❌ |
+<!-- vale on -->
+
+## Netutils filters
+
+The following Jinja2 filters from <a href="https://netutils.readthedocs.io">Netutils</a> are included within Infrahub.
+<!-- vale off -->
+| Name | Trusted |
+|----------|----------|
+| abbreviated_interface_name | ✅ |
+| abbreviated_interface_name_list | ✅ |
+| asn_to_int | ✅ |
+| bits_to_name | ✅ |
+| bytes_to_name | ✅ |
+| canonical_interface_name | ✅ |
+| canonical_interface_name_list | ✅ |
+| cidr_to_netmask | ✅ |
+| cidr_to_netmaskv6 | ✅ |
+| clean_config | ✅ |
+| compare_version_loose | ✅ |
+| compare_version_strict | ✅ |
+| config_compliance | ✅ |
+| config_section_not_parsed | ✅ |
+| delimiter_change | ✅ |
+| diff_network_config | ✅ |
+| feature_compliance | ✅ |
+| find_unordered_cfg_lines | ✅ |
+| fqdn_to_ip | ❌ |
+| get_all_host | ❌ |
+| get_broadcast_address | ✅ |
+| get_first_usable | ✅ |
+| get_ips_sorted | ✅ |
+| get_nist_urls | ✅ |
+| get_nist_vendor_platform_urls | ✅ |
+| get_oui | ✅ |
+| get_peer_ip | ✅ |
+| get_range_ips | ✅ |
+| get_upgrade_path | ✅ |
+| get_usable_range | ✅ |
+| hash_data | ✅ |
+| int_to_asdot | ✅ |
+| interface_range_compress | ✅ |
+| interface_range_expansion | ✅ |
+| ip_addition | ✅ |
+| ip_subtract | ✅ |
+| ip_to_bin | ✅ |
+| ip_to_hex | ✅ |
+| ipaddress_address | ✅ |
+| ipaddress_interface | ✅ |
+| ipaddress_network | ✅ |
+| is_classful | ✅ |
+| is_fqdn_resolvable | ❌ |
+| is_ip | ✅ |
+| is_ip_range | ✅ |
+| is_ip_within | ✅ |
+| is_netmask | ✅ |
+| is_network | ✅ |
+| is_reversible_wildcardmask | ✅ |
+| is_valid_mac | ✅ |
+| longest_prefix_match | ✅ |
+| mac_normalize | ✅ |
+| mac_to_format | ✅ |
+| mac_to_int | ✅ |
+| mac_type | ✅ |
+| name_to_bits | ✅ |
+| name_to_bytes | ✅ |
+| name_to_name | ✅ |
+| netmask_to_cidr | ✅ |
+| netmask_to_wildcardmask | ✅ |
+| normalise_delimiter_caret_c | ✅ |
+| paloalto_panos_brace_to_set | ✅ |
+| paloalto_panos_clean_newlines | ✅ |
+| regex_findall | ❌ |
+| regex_match | ❌ |
+| regex_search | ❌ |
+| regex_split | ❌ |
+| regex_sub | ❌ |
+| sanitize_config | ✅ |
+| section_config | ✅ |
+| sort_interface_list | ✅ |
+| split_interface | ✅ |
+| uptime_seconds_to_string | ✅ |
+| uptime_string_to_seconds | ✅ |
+| version_metadata | ✅ |
+| vlanconfig_to_list | ✅ |
+| vlanlist_to_config | ✅ |
+| wildcardmask_to_netmask | ✅ |
+<!-- vale on -->

docs/sidebars-python-sdk.ts

@@ -38,6 +38,7 @@ const sidebars: SidebarsConfig = {
           label: 'Reference',
           items: [
             'reference/config',
+            'reference/templating',
           ],
         },
       ],

infrahub_sdk/ctl/cli_commands.py

@@ -9,7 +9,6 @@
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Callable, Optional
 
-import jinja2
 import typer
 import ujson
 from rich.console import Console
@@ -18,7 +17,6 @@
 from rich.panel import Panel
 from rich.pretty import Pretty
 from rich.table import Table
-from rich.traceback import Traceback
 
 from .. import __version__ as sdk_version
 from ..async_typer import AsyncTyper
@@ -31,7 +29,7 @@
 from ..ctl.generator import run as run_generator
 from ..ctl.menu import app as menu_app
 from ..ctl.object import app as object_app
-from ..ctl.render import list_jinja2_transforms
+from ..ctl.render import list_jinja2_transforms, print_template_errors
 from ..ctl.repository import app as repository_app
 from ..ctl.repository import get_repository_config
 from ..ctl.schema import app as schema_app
@@ -44,8 +42,9 @@
 )
 from ..ctl.validate import app as validate_app
 from ..exceptions import GraphQLError, ModuleImportError
-from ..jinja2 import identify_faulty_jinja_code
 from ..schema import MainSchemaTypesAll, SchemaRoot
+from ..template import Jinja2Template
+from ..template.exceptions import JinjaTemplateError
 from ..utils import get_branch, write_to_file
 from ..yaml import SchemaFile
 from .exporter import dump
@@ -168,43 +167,28 @@ async def run(
         raise typer.Abort(f"Unable to Load the method {method} in the Python script at {script}")
 
     client = initialize_client(
-        branch=branch, timeout=timeout, max_concurrent_execution=concurrent, identifier=module_name
+        branch=branch,
+        timeout=timeout,
+        max_concurrent_execution=concurrent,
+        identifier=module_name,
     )
     func = getattr(module, method)
     await func(client=client, log=log, branch=branch, **variables_dict)
 
 
-def render_jinja2_template(template_path: Path, variables: dict[str, str], data: dict[str, Any]) -> str:
-    if not template_path.is_file():
-        console.print(f"[red]Unable to locate the template at {template_path}")
-        raise typer.Exit(1)
-
-    templateLoader = jinja2.FileSystemLoader(searchpath=".")
-    templateEnv = jinja2.Environment(loader=templateLoader, trim_blocks=True, lstrip_blocks=True)
-    template = templateEnv.get_template(str(template_path))
-
+async def render_jinja2_template(template_path: Path, variables: dict[str, Any], data: dict[str, Any]) -> str:
+    variables["data"] = data
+    jinja_template = Jinja2Template(template=Path(template_path), template_directory=Path())
     try:
-        rendered_tpl = template.render(**variables, data=data)  # type: ignore[arg-type]
-    except jinja2.TemplateSyntaxError as exc:
-        console.print("[red]Syntax Error detected on the template")
-        console.print(f"[yellow]  {exc}")
-        raise typer.Exit(1) from exc
-
-    except jinja2.UndefinedError as exc:
-        console.print("[red]An error occurred while rendering the jinja template")
-        traceback = Traceback(show_locals=False)
-        errors = identify_faulty_jinja_code(traceback=traceback)
-        for frame, syntax in errors:
-            console.print(f"[yellow]{frame.filename} on line {frame.lineno}\n")
-            console.print(syntax)
-        console.print("")
-        console.print(traceback.trace.stacks[0].exc_value)
+        rendered_tpl = await jinja_template.render(variables=variables)
+    except JinjaTemplateError as exc:
+        print_template_errors(error=exc, console=console)
         raise typer.Exit(1) from exc
 
     return rendered_tpl
 
 
-def _run_transform(
+async def _run_transform(
     query_name: str,
     variables: dict[str, Any],
     transform_func: Callable,
@@ -227,7 +211,11 @@ def _run_transform(
 
     try:
         response = execute_graphql_query(
-            query=query_name, variables_dict=variables, branch=branch, debug=debug, repository_config=repository_config
+            query=query_name,
+            variables_dict=variables,
+            branch=branch,
+            debug=debug,
+            repository_config=repository_config,
         )
 
         # TODO: response is a dict and can't be printed to the console in this way.
@@ -249,15 +237,15 @@ def _run_transform(
         raise typer.Abort()
 
     if asyncio.iscoroutinefunction(transform_func):
-        output = asyncio.run(transform_func(response))
+        output = await transform_func(response)
     else:
         output = transform_func(response)
     return output
 
 
 @app.command(name="render")
 @catch_exception(console=console)
-def render(
+async def render(
     transform_name: str = typer.Argument(default="", help="Name of the Python transformation", show_default=False),
     variables: Optional[list[str]] = typer.Argument(
         None, help="Variables to pass along with the query. Format key=value key=value."
@@ -289,7 +277,7 @@ def render(
     transform_func = functools.partial(render_jinja2_template, transform_config.template_path, variables_dict)
 
     # Query GQL and run the transform
-    result = _run_transform(
+    result = await _run_transform(
         query_name=transform_config.query,
         variables=variables_dict,
         transform_func=transform_func,
@@ -410,7 +398,10 @@ def version() -> None:
 
 @app.command(name="info")
 @catch_exception(console=console)
-def info(detail: bool = typer.Option(False, help="Display detailed information."), _: str = CONFIG_PARAM) -> None:  # noqa: PLR0915
+def info(  # noqa: PLR0915
+    detail: bool = typer.Option(False, help="Display detailed information."),
+    _: str = CONFIG_PARAM,
+) -> None:
     """Display the status of the Python SDK."""
 
     info: dict[str, Any] = {
@@ -476,10 +467,14 @@ def info(detail: bool = typer.Option(False, help="Display detailed information."
         infrahub_info = Table(show_header=False, box=None)
         if info["user_info"]:
             infrahub_info.add_row("User:", info["user_info"]["AccountProfile"]["display_label"])
-            infrahub_info.add_row("Description:", info["user_info"]["AccountProfile"]["description"]["value"])
+            infrahub_info.add_row(
+                "Description:",
+                info["user_info"]["AccountProfile"]["description"]["value"],
+            )
             infrahub_info.add_row("Status:", info["user_info"]["AccountProfile"]["status"]["label"])
             infrahub_info.add_row(
-                "Number of Groups:", str(info["user_info"]["AccountProfile"]["member_of_groups"]["count"])
+                "Number of Groups:",
+                str(info["user_info"]["AccountProfile"]["member_of_groups"]["count"]),
             )
 
             if groups := info["groups"]: