Transform community lists into type/value format (#2851)

This is a preparatory step for the introduction of extended/large
community lists. The final format of the community lists is
unchanged, but the transformation into 'type'/'value' format is
done earlier in the process as we'll use 'type'/'value' format
as the canonical format for the more complex community lists.

The final data structure is unchanged, which means that this
refactoring does not impact the device configuration templates
or any user-side functionality depending on the transformed
data structures.

The 'rp-clist-expansion' transformation test was modified to
include the single-entry box format and the canonical
type/value format to test all possible lab topology formats
that can be used to specify a standard clist.

Author: ipspace

netsim/modules/routing.yml

@@ -39,9 +39,13 @@ attributes:
       type: dict
       _keytype: id
       _subtype:
-        type: list
-        make_list: True
-        _subtype: community_entry
+        type: dict
+        _keys:
+          type: { type: str, valid_values: [ standard, extended, large ]}
+          value:
+            type: list
+            make_list: True
+            _subtype: community_entry
     static:
       type: dict
       _keytype: id

netsim/modules/routing/__init__.py

@@ -37,7 +37,7 @@
     'import' : import_routing_object,
     'check'  : check_routing_object },
   'community': {
-    'import' : clist.import_community_list,
+    'import' : import_routing_object,
     'check'  : check_routing_object },
   'static': {
     'start'  : static.include_global_static_routes
@@ -63,6 +63,7 @@
   'community':
     { 'namespace': 'routing.community',
       'object'   : 'BGP community filter',
+      'list_attr': 'value',
       'callback' : aspath.normalize_aspath_entry },
 }
 
@@ -85,6 +86,7 @@ def normalize_routing_data(r_object: Box, topo_object: bool = False, o_name: str
           o_dict=r_object.get(dp['namespace'],None),
           o_type=dp['object'],
           normalize_callback=dp['callback'],
+          list_attr=dp.get('list_attr',None),
           topo_object=topo_object)
   except Exception as ex:
     log.warning(

netsim/modules/routing/clist.py

@@ -12,20 +12,6 @@
 
 from ...augment import devices
 from ...utils import log
-from .normalize import (
-  import_routing_object,
-)
-
-"""
-import_community_list -- has to check whether the target clist was already fixed, which means
-the transform_dispatch has already been called
-"""
-def import_community_list(pname: str,o_name: str,node: Box,topology: Box) -> typing.Optional[list]:
-  clist = node.get(f'routing.{o_name}.{pname}',None)
-  if isinstance(clist,Box):
-    return None
-
-  return import_routing_object(pname,o_name,node,topology)
 
 """
 replace_community_delete:
@@ -55,7 +41,7 @@ def replace_community_delete(node: Box, p_name: str, p_entry: Box, topology: Box
 
   if clist:
     cname = f"DEL_{p_name}_{p_entry.sequence}"
-    node.routing.community[cname] = { 'action': 'permit', 'cl_type': 'standard', 'value': clist }
+    node.routing.community[cname] = { 'type': 'standard', 'cl_type': 'standard', 'value': clist }
     p_entry.delete.community.list = cname
 
   for kw in ['standard','large','extended']:
@@ -73,6 +59,8 @@ class MissingQuotes(Exception):
   pass
 
 def fix_clist_entry(e_clist: Box, topology: Box) -> None:
+  if '_value' in e_clist:                                   # Is this entry already in the expected format?
+    return
   if 'regexp' in e_clist:                                   # Do we know we have a regexp?
     e_clist._value = e_clist.regexp                         # Copy regular expression into a common variable
     return
@@ -101,14 +89,8 @@ def fix_clist_entry(e_clist: Box, topology: Box) -> None:
 """
 def expand_community_list(p_name: str,o_name: str,node: Box,topology: Box) -> typing.Optional[list]:
   p_clist = node.routing[o_name][p_name]                    # Shortcut pointer to current community list
-  if isinstance(p_clist,Box) and 'value' in p_clist:        # Have we already transformed this clist?
-    return None
-
-  node.routing[o_name][p_name] = {                          # Move the permit/deny list into 'value
-    'value': p_clist
-  }
-
-  p_clist = node.routing[o_name][p_name]                    # ... and fetch the new shortcut pointer
+  if 'type' not in p_clist:
+    p_clist.type = 'standard'                               # Assume the clist filter for standard communities
   regexp = False                                            # Figure out whether we need expanded clist
   for (p_idx,p_entry) in enumerate(p_clist.value):
     try:

netsim/modules/routing/normalize.py

@@ -36,6 +36,7 @@ def normalize_routing_objects(
       o_dict: typing.Optional[Box],
       o_type: str,
       normalize_callback: typing.Callable,
+      list_attr: typing.Optional[str] = None,
       topo_object: bool = False) -> None:
 
   if o_dict is None:                                        # Nothing to do, I'm OK with that ;)
@@ -53,10 +54,25 @@ def normalize_routing_objects(
           module='routing')
       continue
 
-    if not isinstance(o_dict[o_name],list):                 # Object not a list? Let's make it one ;)
-      o_dict[o_name] = [ o_dict[o_name] ]
+    o_value = o_dict[o_name]
 
-    normalize_routing_object(o_dict[o_name],normalize_callback)
+    # Convert a non-list value to list only if it's not a box with an attribute
+    # that is expected to contain a list
+    #
+    if not isinstance(o_value,Box) or list_attr is None or list_attr not in o_value:
+      if not isinstance(o_dict[o_name],list):                 # Object not a list? Let's make it one ;)
+        o_dict[o_name] = [ o_dict[o_name] ]
+
+    if list_attr:                                             # Are we expected to have a box with a list?
+      if isinstance(o_dict[o_name],list):                     # ... but only have a list (maybe created above)?
+        o_list = o_dict[o_name]                               # Save the list value for the next step
+        o_dict[o_name] = { list_attr: o_list }                # Convert it to the expected box format
+      else:                                                   # Otherwise the data structure must be a box with a
+        o_list = o_dict[o_name][list_attr]                    # ... list attribute or it would have been converted
+    else:                                                     # Or maybe this object is a simple list?
+      o_list = o_dict[o_name]                                 # ... so store the list value, I guess
+
+    normalize_routing_object(o_list,normalize_callback)       # Finally, normalize the list
 
 """
 check_routing_object: validate that a device supports the requested routing object
@@ -77,7 +93,13 @@ def check_routing_object(p_name: str,o_type: str, node: Box,topology: Box) -> bo
 
 Return merged routing object or None if the node routing object has not been modified or does not exist
 """
-def import_routing_object(pname: str,o_name: str,node: Box,topology: Box) -> typing.Optional[list]:
+def import_routing_object(
+      pname: str,
+      o_name: str,
+      node: Box,
+      topology: Box) -> typing.Optional[list]:
+
+  from . import normalize_dispatch
   topo_pdata = topology.get(f'routing.{o_name}',{})
 
   # First check whether the node routing object is missing or set to None (no value)
@@ -99,6 +121,11 @@ def import_routing_object(pname: str,o_name: str,node: Box,topology: Box) -> typ
 
   np_data = node.routing[o_name][pname]                     # Prepare for merge: get node- and global entries
   tp_data = topo_pdata[pname]
+  list_attr = normalize_dispatch[o_name].get('list_attr',None)
+  if list_attr:                                             # Are we storing the list value in a dict attribute?
+    np_data = np_data[list_attr]                            # ... if we do, work with those values
+    tp_data = tp_data[list_attr]
+
   sqlist  = [ pe.sequence for pe in np_data ]               # Get the list of sequence numbers from the local policy
 
   # Now get global entries that are missing in the local policy,
@@ -109,8 +136,12 @@ def import_routing_object(pname: str,o_name: str,node: Box,topology: Box) -> typ
     return None
 
   np_data = sorted(np_data + tp_add, key= lambda x: x.sequence)
-  node.routing[o_name][pname] = np_data
-  return np_data
+  if list_attr:                                             # Finally, the sorted list must be stored
+    node.routing[o_name][pname][list_attr] = np_data        # ... in the expected dict attribute
+  else:
+    node.routing[o_name][pname] = np_data                   # ... or replace the original list
+
+  return node.routing[o_name][pname]
 
 """
 Import or merge global routing policies into node routing policies

netsim/modules/routing/policy.py

@@ -192,7 +192,10 @@ def import_policy_filters(pname: str, o_name: str, node: Box, topology: Box) ->
         f_import = import_routing_object(p_entry[kw],r_object,node,topology)
         if f_import:                                        # If we imported any new data...
           if r_object in normalize_dispatch:                # ... normalize the filter entries
-            normalize_routing_object(f_import,normalize_dispatch[r_object]['callback'])
+            if 'list_attr' in normalize_dispatch[r_object]: # Do we have list entries in a box?
+              f_import = f_import[normalize_dispatch[r_object]['list_attr']]
+            if f_import:                                    # Is there anything to process?
+              normalize_routing_object(f_import,normalize_dispatch[r_object]['callback'])
           if r_object in import_dispatch and 'check' in import_dispatch[r_object]:
             import_dispatch[r_object]['check'](p_entry[kw],r_object,node,topology)
           if r_object in transform_dispatch:                # ... and transform the filter into its final form

tests/topology/expected/rp-clist-expansion.yml

@@ -6,10 +6,11 @@ groups:
       routing:
         community:
           cg1:
-          - action: permit
-            path:
-            - 65000:100
-            sequence: 10
+            value:
+            - action: permit
+              path:
+              - 65000:100
+              sequence: 10
 input:
 - topology/input/rp-clist-expansion.yml
 - package:topology-defaults.yml
@@ -43,13 +44,15 @@ nodes:
         cg1:
           cl_type: standard
           regexp: ''
+          type: standard
           value:
           - _value: 65000:100
             action: permit
             sequence: 10
         cl2:
           cl_type: standard
           regexp: ''
+          type: standard
           value:
           - _value: 65000:100
             action: permit
@@ -60,6 +63,7 @@ nodes:
         cl3:
           cl_type: expanded
           regexp: regexp
+          type: standard
           value:
           - _value: _65000:10[1-2]_
             action: permit
@@ -68,6 +72,7 @@ nodes:
         cl4:
           cl_type: standard
           regexp: ''
+          type: standard
           value:
           - _value: 65000:104
             action: permit
@@ -81,6 +86,7 @@ nodes:
         cl5:
           cl_type: expanded
           regexp: regexp
+          type: standard
           value:
           - _value: 65000:100 65001:100
             action: deny
@@ -95,6 +101,7 @@ nodes:
         cl6:
           cl_type: expanded
           regexp: regexp
+          type: standard
           value:
           - _value: 65000:100 65001:100
             action: deny
@@ -106,6 +113,7 @@ nodes:
         cl7:
           cl_type: standard
           regexp: ''
+          type: standard
           value:
           - _value: 65000:106
             action: permit
@@ -128,15 +136,19 @@ provider: libvirt
 routing:
   community:
     cl4:
-    - action: permit
-      path:
-      - 65000:104
-      sequence: 10
+      value:
+      - action: permit
+        path:
+        - 65000:104
+        sequence: 10
     cl5:
-    - action: permit
-      path: 65000:100
-      sequence: 100
+      value:
+      - action: permit
+        path:
+        - 65000:100
+        sequence: 100
     cl7:
-    - _value: 65000:106
-      action: permit
-      sequence: 10
+      value:
+      - _value: 65000:106
+        action: permit
+        sequence: 10

tests/topology/input/rp-clist-expansion.yml

@@ -18,7 +18,9 @@ routing:
     - sequence: 100
       action: permit
       path: 65000:100
-    cl7: 65000:106
+    cl7:
+      action: permit
+      path: 65000:106
 
 nodes:
   r1:
@@ -41,9 +43,11 @@ nodes:
           path: [65000:103, 65001:103]
           sequence: 30
         cl5:                                      # A mix standard and extended conditions ==> extended
-        - action: deny
-          path: [65000:100, 65001:100]            # first entry is a list of communities
-        - _6510.:307_                             # the second entry is a regexp
+          type: standard
+          value:
+          - action: deny
+            path: [65000:100, 65001:100]          # first entry is a list of communities
+          - _6510.:307_                           # the second entry is a regexp
         cl6:                                      # Permit any at the end forces an extended clist
         - action: deny
           path: [65000:100, 65001:100]