Make netlab Jinja2 handling more Ansible-like

* The undefined variables are handled with a derived Undefined type
  that behaves like Ansible variables. For example 'x.y is defined'
  returns false (instead of crashing) if x is not defined.
* All Jinja2 filters are defined as having 'ansible.utils.X' synonim.

With these enhancements, we can revert the changes made to Bird and
Linux templates

Also: fix the ipaddr-like templates

* ipv4() and ipv6() should accept zero arguments (and act like a
  "select addresses from list" filter)
* ipaddr(int) should return a prefix, not an IP address

Author: ipspace

netsim/ansible/templates/initial/bird-clab.j2

@@ -1,7 +1,7 @@
 set -x
 {% include 'linux-clab.j2' +%}
 #
-{% if loopback is defined and loopback.ipv6 is defined %}
+{% if loopback.ipv6 is defined %}
 set +e
 ip addr add fe80::1/64 dev lo scope link
 {% endif %}

netsim/ansible/templates/vlan/linux-clab.j2

@@ -1,7 +1,6 @@
 {% include 'frr.j2' +%}
 {% from "initial/linux/vanilla-ifconfig.j2" import ifconfig %}
 {% for intf in interfaces 
-     if intf.type == 'svi' or 
-        intf.vlan is defined and intf.vlan.mode|default('') == 'route' %}
+     if intf.type == 'svi' or intf.vlan.mode|default('') == 'route' %}
 {{   ifconfig(intf) }}
 {% endfor %}

netsim/utils/filters.py

@@ -14,6 +14,7 @@
 import typing
 
 import netaddr
+from jinja2.runtime import StrictUndefined
 
 
 def ipaddr_filter(
@@ -49,7 +50,7 @@ def j2_ipaddr(
 
   addr = netaddr.IPNetwork(value)
   if isinstance(arg,int):
-    return str(addr[arg])
+    return str(addr[arg]) + "/" + str(addr.prefixlen)
 
   if arg in MAP_IPADDR:
     arg = MAP_IPADDR[arg]
@@ -62,10 +63,10 @@ def j2_ipaddr(
 
   raise ValueError(f'Invalid argument {arg} passed to built-in ipaddr filter')
 
-def j2_ipv4(value: typing.Any, arg: typing.Union[int,str]) -> typing.Union[list,str]:
+def j2_ipv4(value: typing.Any, arg: typing.Union[int,str] = '') -> typing.Union[list,str]:
   return j2_ipaddr(value,arg,4)
 
-def j2_ipv6(value: typing.Any, arg: typing.Union[int,str]) -> typing.Union[list,str]:
+def j2_ipv6(value: typing.Any, arg: typing.Union[int,str] = '') -> typing.Union[list,str]:
   return j2_ipaddr(value,arg,6)
 
 # Format MAC addresses in Cisco/Unix/... format
@@ -85,3 +86,58 @@ def j2_hwaddr(value: typing.Any, format: str = '') -> str:
       raise ValueError(f'{value} is not a valid MAC address and cannot be formatted as {format}')
     else:                                         # Otherwise it was a filter query, return empty string
       return ''
+
+class j2_Undefined(StrictUndefined):
+  """
+  Mimics Ansible's undefined variable handling in Jinja2 templates.
+  Accessing attributes or items of an undefined variable returns another undefined,
+  and expressions like 'x.y is defined' return False when x is undefined,
+  rather than raising an error.
+  """
+
+  # Override attribute access
+  def __getattr__(self, name: typing.Any) -> typing.Any:
+      return j2_Undefined(name=name, hint=self._undefined_hint)
+
+  # Override item access
+  def __getitem__(self, key: typing.Any) -> typing.Any:
+      return j2_Undefined(name=key, hint=self._undefined_hint)
+
+  # Override boolean evaluation
+  def __bool__(self) -> bool:
+      return False
+
+  # Override is defined
+  @property
+  def defined(self) -> bool:
+      return False
+
+UTILS_FILTERS: dict = {
+  'ipaddr': j2_ipaddr,
+  'ipv4': j2_ipv4,
+  'ipv6': j2_ipv6,
+  'hwaddr': j2_hwaddr
+}
+
+"""
+Simplified 'flatten' filter -- returns a flattened list
+"""
+def bi_flatten(mylist: typing.Any, levels: typing.Optional[int] = None) -> list:
+  if not isinstance(mylist,list):
+    return [ mylist ]
+
+  result = []
+  for value in mylist:
+    if isinstance(value,list):
+      if (levels is None or levels >= 1):
+        result.extend(bi_flatten(value,None if levels is None else levels - 1))
+      else:
+        result.extend(value)
+    else:
+      result.append(value)
+
+  return result      
+
+BUILTIN_FILTERS: dict = {
+  'flatten': bi_flatten
+}

netsim/utils/templates.py

@@ -7,7 +7,7 @@
 import typing
 
 from box import Box
-from jinja2 import Environment, FileSystemLoader, StrictUndefined, make_logging_undefined
+from jinja2 import Environment, FileSystemLoader
 
 from ..augment import devices
 from ..outputs import common as outputs_common
@@ -18,11 +18,13 @@
 
 
 def add_j2_filters(ENV: Environment) -> None:
-  for fname in dir(filters):                      # Get all attributes of the "filters" module
-    if not fname.startswith('j2_'):               # Filters have to start with 'j2_' prefix
-      continue
-    fcode = getattr(filters,fname)                # Get a pointer to filter function
-    ENV.filters[fname.replace('j2_','')] = fcode  # And define a new Jinja2 filter
+  for fname,fcode in filters.UTILS_FILTERS.items():         # Iterate over internal filter definitions
+    ENV.filters[fname] = fcode                              # ... emulating the ansible.utils filters we use
+    ENV.filters[f'ansible.utils.{fname}'] = fcode           # ... define simple filter name and its Ansible FQFN
+
+  for fname,fcode in filters.BUILTIN_FILTERS.items():       # Do the same for ansible.builtin filters we use
+    ENV.filters[fname] = fcode
+    ENV.filters[f'ansible.builtin.{fname}'] = fcode
 
 """
 Render a Jinja2 template
@@ -41,7 +43,7 @@ def add_j2_filters(ENV: Environment) -> None:
 def get_jinja2_env_for_path(template_path: tuple) -> Environment:
   ENV = Environment(loader=FileSystemLoader(template_path), \
           trim_blocks=True,lstrip_blocks=True, \
-          undefined=make_logging_undefined(base=StrictUndefined))
+          undefined=filters.j2_Undefined)
   add_j2_filters(ENV)
   return ENV