Static (topology) 'tc' functionality for clab and libvirt (#2626)

Implements #1387

Author: ipspace

docs/links.md

@@ -442,6 +442,23 @@ A lab device could be a networking device or a host[^HOST]. Links with attached
 
 [^NOPASS]: To turn a link with hosts attached into a transit link, set link **role** to **lan** (or any other role).
 
+(links-netem)=
+## Link Impairment
+
+_netlab_ can configure the Linux **netem** queuing discipline to introduce link impairment on point-to-point or LAN links[^tc]. The **tc** link- or interface attribute allows you to configure these QoS parameters:
+
+* **tc.delay** -- transmission delay specified in milliseconds (`ms`) or seconds (`s`)
+* **tc.jitter** -- jitter (using the same units as **tc.delay**)
+* **tc.loss** -- loss percentage (0..100)
+* **tc.corrupt** -- packet corruption percentage
+* **tc.duplicate** -- packet duplication percentage
+* **tc.reorder** -- packet reordering percentage
+* **tc.rate** -- rate throttling (in kbps). Delays packets to emulate a fixed link speed
+
+[^tc]: This feature works with *[clab](lab-clab)* and *[libvirt](lab-libvirt)* providers. *libvirt* point-to-point links are converted to LAN links (using a Linux bridge), which means that you cannot use **tc** together with link aggregation on inter-VM links.
+
+While you could configure **tc** parameters on individual interfaces, the **netem** queuing discipline applies them only to outgoing traffic. You should therefore configure **tc** parameters on links to ensure the same parameters are applied to all interfaces connected to the link.
+
 (links-bridge)=
 ## Bridge Names
 

netsim/cli/up.py

@@ -373,6 +373,8 @@ def run_up(cli_args: typing.List[str]) -> None:
     recreate_secondary_config(topology,p_provider,s_provider)
     start_provider_lab(topology,p_provider,s_provider)
 
+  providers.execute_tc_commands(topology)
+
   try:
     if args.reload:
       reload_saved_config(args,topology)

netsim/defaults/attributes.yml

@@ -65,6 +65,14 @@ link:                       # Global link attributes
     onlink: bool
   role: id
   pool: id
+  tc:
+    delay: time
+    jitter: time
+    loss: { type: float, min_value: 0, max_value: 100 }
+    corrupt: { type: float, min_value: 0, max_value: 100 }
+    duplicate: { type: float, min_value: 0, max_value: 100 }
+    reorder: { type: float, min_value: 0, max_value: 100 }
+    rate: { type: int, min_value: 1 }
   type: { type: str, valid_values: [ lan, p2p, stub, loopback, tunnel, vlan_member ] }
   unnumbered: bool
   interfaces:

netsim/providers/__init__.py

@@ -233,6 +233,13 @@ def get_lab_status(self) -> Box:
   def get_node_name(self, node: str, topology: Box) -> str:
     return node
 
+  def set_tc(self, node: Box, topology: Box, intf: Box) -> None:
+    n_provider = devices.get_provider(node,topology.defaults)
+    log.warning(
+      text=f'tc is not supported (node {node.name}, link {intf.name})',
+      module=n_provider,
+      flag=f'{n_provider}.tc')
+
   """
   Generic provider pre-transform processing: Mark multi-provider links
   """
@@ -329,10 +336,10 @@ def execute(hook: str, topology: Box) -> None:
 """
 Execute a node-level provider hook
 """
-def execute_node(hook: str, node: Box, topology: Box) -> typing.Any:
+def execute_node(hook: str, node: Box, topology: Box, **kwargs: typing.Any) -> typing.Any:
   node_provider = devices.get_provider(node,topology.defaults)
   p_module = get_provider_module(topology,node_provider)
-  return p_module.call(hook,node,topology)
+  return p_module.call(hook,node,topology,**kwargs)
 
 """
 Mark all nodes and links with relevant provider(s)
@@ -439,3 +446,51 @@ def validate_mgmt_ip(
         f'Management {af} address of node {node.name} ({n_mgmt[af]}) is not part of the management subnet',
         category=log.IncorrectValue,
         module=provider)
+
+"""
+Execute tc commands
+"""
+def execute_tc_commands(topology: Box) -> None:
+  for ndata in topology.nodes.values():
+    for intf in ndata.interfaces:
+      if 'tc' not in intf:
+        continue
+      execute_node('set_tc',node=ndata,topology=topology,intf=intf)
+
+"""
+Apply tc netem parameters to the specified interface
+"""
+NETEM_KW_MAP = {
+  'delay': ' delay {delay}ms {jitter}ms'
+}
+
+NETEM_SIMPLE_KW = [ 'loss', 'corrupt', 'duplicate', 'reorder', 'rate' ]
+
+def tc_netem_set(intf: str, tc_data: Box, pfx: str = '') -> typing.Union[str,bool]:
+  global NETEM_KW_MAP,NETEM_SIMPLE_KW
+  from ..cli import external_commands
+
+  netem_params = ''
+  if 'jitter' in tc_data and 'delay' not in tc_data:        # Delay and jitter have to be specified
+    tc_data.delay = 0                                       # ... in a single netem parameter
+  if 'delay' in tc_data and 'jitter' not in tc_data:        # ... so we have to ensure both of them
+    tc_data.jitter = 0                                      # ... are set at the same time
+
+  for kw in tc_data:
+    if kw in NETEM_SIMPLE_KW:
+      netem_params += f' {kw} {tc_data[kw]}'
+    elif kw in NETEM_KW_MAP:
+      netem_params += NETEM_KW_MAP[kw].format(**tc_data)
+
+  if 'sudo' not in pfx:
+    pfx = 'sudo '+pfx
+  qdisc = external_commands.run_command(
+    cmd=pfx + f' tc qdisc show dev {intf}',
+    ignore_errors=True,return_stdout=True,check_result=True)
+  if isinstance(qdisc,str) and 'noqueue' not in qdisc:
+    external_commands.run_command(
+      cmd=pfx + f' tc qdisc del dev {intf} root',
+      ignore_errors=True,return_stdout=True,check_result=True)
+
+  status = external_commands.run_command(pfx + f' tc qdisc add dev {intf} root netem'+netem_params)
+  return netem_params if status else False

netsim/providers/clab.py

@@ -13,7 +13,7 @@
 from ..data import append_to_list, filemaps, get_empty_box
 from ..data.types import must_be_dict
 from ..utils import linuxbridge, log, strings
-from . import _Provider, get_provider_forwarded_ports, node_add_forwarded_ports, validate_mgmt_ip
+from . import _Provider, get_provider_forwarded_ports, node_add_forwarded_ports, tc_netem_set, validate_mgmt_ip
 
 
 def list_bridges( topology: Box ) -> typing.Set[str]:
@@ -278,3 +278,17 @@ def capture_command(self, node: Box, topology: Box, args: argparse.Namespace) ->
     cmd = strings.eval_format_list(cmd,{'intf': args.intf})
     node_name = self.get_node_name(node.name,topology)
     return strings.string_to_list(f'sudo ip netns exec {node_name}') + cmd
+
+  def set_tc(self, node: Box, topology: Box, intf: Box) -> None:
+    c_name = self.get_node_name(node.name,topology)
+    c_intf = intf.get('clab.name',intf.ifname)
+    netns = 'sudo ip netns exec ' + c_name
+    status = tc_netem_set(intf=c_intf,tc_data=intf.tc,pfx=netns)
+    if status:
+      log.info(text=f'Traffic control on {node.name} {intf.ifname}:{status}')
+    else:
+      log.error(
+        text=f'Failed to deploy tc policy on {node.name} (container {c_name}) interface {c_intf}',
+        module='clab',
+        skip_header=True,
+        category=log.ErrorAbort)

netsim/providers/libvirt.py

@@ -18,7 +18,7 @@
 from ..data import get_box, get_empty_box, types
 from ..utils import files as _files
 from ..utils import linuxbridge, log, strings
-from . import _Provider, get_provider_forwarded_ports, node_add_forwarded_ports, validate_mgmt_ip
+from . import _Provider, get_provider_forwarded_ports, node_add_forwarded_ports, tc_netem_set, validate_mgmt_ip
 
 LIBVIRT_MANAGEMENT_NETWORK_NAME  = "vagrant-libvirt"
 LIBVIRT_MANAGEMENT_BRIDGE_NAME   = "libvirt-mgmt"
@@ -284,8 +284,22 @@ def pre_transform(self, topology: Box) -> None:
         if not 'public' in l.libvirt:                            # ... but no 'public' libvirt attr
           l.libvirt.public = 'bridge'                            # ... default mode is bridge (MACVTAP)
 
+      """
+      The libvirt links could be modeled as P2P links (using UDP tunnels) or
+      LAN links using a Linux bridge. It's better to use the UDP tunnels, but
+      we must us the Linux bridge if:
+
+      * The link type is 'lan' or 'stub' (set/used elsewhere, also includes
+        hosts connected to links)
+      * The libvirt.provider attribute is set (multi-provider links or external
+        connectivity)
+      * The system defaults say P2P links should be modeled as bridges
+        (used for traffic capture)
+      * The link or any of the interfaces has the 'tc' parameter
+      """
       must_be_lan = l.get('libvirt.provider',None) and 'vlan' not in l.type
       must_be_lan = must_be_lan or (p2p_bridge and l.get('type','p2p') == 'p2p')
+      must_be_lan = must_be_lan or 'tc' in l or [ intf for intf in l.interfaces if 'tc' in intf ]
       if must_be_lan:
         l.type = 'lan'
         if not 'bridge' in l:
@@ -482,37 +496,76 @@ def validate_node_image(self, node: Box, topology: Box) -> None:
         f"'vagrant box add <url>' command to add it, or use this recipe to build it:",
         dp_data.build ])
 
-  def capture_command(self, node: Box, topology: Box, args: argparse.Namespace) -> typing.Optional[list]:
-    intf = [ intf for intf in node.interfaces if intf.ifname == args.intf ][0]
-    if intf.get('libvirt.type',None) == 'tunnel':
+  def get_linux_intf(
+        self,
+        node: Box,
+        topology: Box,
+        ifname: str,
+        op: str,
+        hint: str,
+        exit_on_error: bool = True) -> typing.Optional[str]:
+
+    intf = [ intf for intf in node.interfaces if intf.ifname == ifname ][0]
+    if intf.get('libvirt.type',None) == 'tunnel' or 'bridge' not in intf:
       log.error(
-        f'Cannot perform packet capture on libvirt point-to-point links',
+        f'Cannot perform {op} on libvirt point-to-point links',
         category=log.FatalError,
         module='libvirt',
         skip_header=True,
-        exit_on_error=True,
-        hint='capture')
+        exit_on_error=exit_on_error,
+        hint=hint)
+      return None
 
     domiflist = external_commands.run_command(
                   ['virsh','domiflist',f'{topology.name}_{node.name}'],
                   check_result=True,
                   return_stdout=True)
     if not isinstance(domiflist,str):
+      log.error(
+        f'Cannot get the list of libvirt interface for node {node.name}',
+        category=log.FatalError,
+        module='libvirt',
+        skip_header=True,
+        exit_on_error=exit_on_error)
       return None
 
     for intf_line in domiflist.split('\n'):
       intf_data = strings.string_to_list(intf_line)
       if len(intf_data) != 5:
         continue
       if intf_data[2] == intf.bridge:
-        cmd = strings.string_to_list(topology.defaults.netlab.capture.command)
-        cmd = strings.eval_format_list(cmd,{'intf': intf_data[0]})
-        return ['sudo'] + cmd
-      
+        return intf_data[0]
+
     log.error(
       f'Cannot find the interface on node {node.name} attached to libvirt network {intf.bridge}',
       category=log.FatalError,
       module='libvirt',
       skip_header=True,
-      exit_on_error=True)
+      exit_on_error=exit_on_error)
     return None
+
+  def capture_command(self, node: Box, topology: Box, args: argparse.Namespace) -> typing.Optional[list]:
+    ifname = self.get_linux_intf(node,topology,args.intf,op='packet capture',hint='capture')
+    if not ifname:
+      return None
+
+    cmd = strings.string_to_list(topology.defaults.netlab.capture.command)
+    cmd = strings.eval_format_list(cmd,{'intf': ifname})
+    return ['sudo'] + cmd
+
+  def set_tc(self, node: Box, topology: Box, intf: Box) -> None:
+    vm_intf = self.get_linux_intf(
+                node,topology,ifname=intf.ifname,
+                op='traffic control',hint='tc',exit_on_error=False)
+    if not vm_intf:
+      return
+
+    status = tc_netem_set(intf=vm_intf,tc_data=intf.tc)
+    if status:
+      log.info(text=f'Traffic control on {node.name} {intf.ifname}:{status}')
+    else:
+      log.error(
+        text=f'Failed to deploy tc policy on {node.name} interface {intf.ifname} (Linux interface {vm_intf})',
+        module='libvirt',
+        skip_header=True,
+        category=log.ErrorAbort)

tests/topology/expected/link-bw.yml

@@ -4,6 +4,7 @@ input:
 links:
 - _linkname: links[1]
   bandwidth: 100000
+  bridge: input_1
   interfaces:
   - ifindex: 1
     ifname: GigabitEthernet0/1
@@ -25,7 +26,7 @@ links:
     jitter: 13
     loss: 0.3
     rate: 2000
-  type: p2p
+  type: lan
 name: input
 nodes:
   e1:
@@ -37,6 +38,7 @@ nodes:
     id: 1
     interfaces:
     - bandwidth: 100000
+      bridge: input_1
       ifindex: 1
       ifname: GigabitEthernet0/1
       ipv4: 192.168.23.1/24
@@ -53,7 +55,7 @@ nodes:
         jitter: 13
         loss: 0.3
         rate: 2000
-      type: p2p
+      type: lan
     loopback:
       ifindex: 0
       ifname: Loopback0
@@ -76,6 +78,7 @@ nodes:
     id: 2
     interfaces:
     - bandwidth: 100000
+      bridge: input_1
       ifindex: 1
       ifname: GigabitEthernet0/1
       ipv4: 192.168.23.2/24
@@ -92,7 +95,7 @@ nodes:
         jitter: 13
         loss: 0.3
         rate: 2000
-      type: p2p
+      type: lan
     loopback:
       ifindex: 0
       ifname: Loopback0

tests/topology/input/link-bw.yml

@@ -1,13 +1,6 @@
 defaults:
   device: iosv
   inventory: dump
-  attributes:
-    link:
-      tc:
-        delay: time
-        jitter: time
-        loss: { type: float, min_value: 0, max_value: 100 }
-        rate: { type: int, min_value: 1 }
 
 nodes:
   e1: