Merge pull request #4982 from hareeshpc/dev/bypass

Use accelerated device as Gateway interface

Author: girishmg

docs/design/gatway-accelerated-interface-configuration.md

@@ -0,0 +1,84 @@
+# Gateway Accelerated Interface Configuration
+
+## Description
+
+To provide hardware acceleration for traffic, both IN and OUT ports need to be a hardware 
+accelerated netdevice backed by the Network Interface Card hardware itself.
+In case of external traffic, when one such port is the external OVS bridge, which for example has the gateway IP, 
+such traffic (like host networking traffic) would not be accelerated.
+Using Switchdev VirtualFunction (VF) or SubFunction (SF) as a gateway interface allows to accelerate these too.
+
+
+## How it works?
+
+Instead of using the gateway interface as the external bridge itself, use a  switchdev VF or SF instead. 
+This is depicted as following:
+
+```
+                         +----------+
+                         |  br-ext  |
+                   +--------+       |
+                   | UPLINK |       |
+                   +--------+       |  patch  +----------+
+                         |          x---------x  br-int  |
+    +--------+     +--------+       |  port   +----------+
+    | NETDEV +-----+   REP  |       |
+    +--------+     +--------+       |
+                         +----------+
+```
+
+Where `UPLINK` is a port on an offloading capable network interface hardware, `NETDEV` is a switchdev function 
+of this port and `REP` is a representor netdevice of the switchdev function. 
+Node/Host IP assigned to `NETDEV` which make OVS to chose `REP` port for external flows instead of the bridge.
+
+
+## How to use?
+
+Gateway accelerated interface can be used in two steps:
+
+a) Creating and configuring the device. 
+See figure above.
+An `UPLINK` device is connected to the OVS external bridge. 
+An existing VF or SF `NETDEV` from the `UPLINK` is first selected as the the Gateway Interface. Its associated 
+representor `REP` is plugged into the OVS external bridge (br-ext). The gateway IP is assigned to this interface 
+instead of the OVS external bridge (br-ext). 
+
+b) Specify `NETDEV` as a gateway interface explicitly via `OVN_GATEWAY_OPTS` environment variable for
+  ovnkube-node container. Example:
+
+```yaml
+            - name: OVN_GATEWAY_OPTS
+              value: "--gateway-accelerated-interface=<<NETDEV>>"
+```
+
+Note that this is mutually exclusive to the `--gateway-interface` flag for GATEWAY_OPTIONS.
+
+c) Set the external-id on the bridge to detect the uplink device correctly. This is useful for instances where,
+the name of the bridge (eg: br-ext) does not use the uplink device (eg: p0) in its name. The uplink can also 
+be a bond device. 
+```bash
+ovs-vsctl br-set-external-id br-ext bridge-uplink p0
+```
+This gives more flexibility in detecting the uplink device in cases where the auto detection fails (like in case of 
+bonded uplinks etc.)
+
+## Verification
+
+Openflow rules added to the external bridge will use this port as the IN/OUT port instead.
+
+Example flows when pf0vf1 is the netdev and pf0vf1_r is the representor
+```bash
+ cookie=0xdeff105, duration=505314.637s, table=0, n_packets=0, n_bytes=0, priority=500,ip,in_port="pf0vf1_r",nw_dst=169.254.0.1 actions=ct(table=5,zone=64002,nat)
+ cookie=0xdeff105, duration=505314.637s, table=0, n_packets=655, n_bytes=129843, priority=500,ip,in_port="pf0vf1_r",nw_dst=10.96.0.0/16 actions=ct(commit,table=2,zone=64001,nat(src=169.254.0.2))
+ cookie=0xdeff105, duration=505314.637s, table=0, n_packets=359877855, n_bytes=531033264511, priority=205,udp,in_port=p0,dl_dst=42:0b:9a:f1:83:b2,tp_dst=6081 actions=output:"pf0vf1_r"
+ cookie=0xdeff105, duration=505314.637s, table=0, n_packets=6252796, n_bytes=775727815, priority=200,udp,in_port="pf0vf1_r",tp_dst=6081 actions=output:p0
+ cookie=0xdeff105, duration=505314.637s, table=0, n_packets=1867752, n_bytes=294547557, priority=100,ip,in_port="pf0vf1_r" actions=ct(commit,zone=64000,exec(load:0x2->NXM_NX_CT_MARK[])),output:p0
+ cookie=0xdeff105, duration=505314.637s, table=0, n_packets=22, n_bytes=1320, priority=10,in_port=p0,dl_dst=42:0b:9a:f1:83:b2 actions=output:"patch-brp0_c-23",output:"pf0vf1_r"
+ cookie=0xdeff105, duration=505314.637s, table=1, n_packets=1313364, n_bytes=669490616, priority=100,ct_state=+est+trk,ct_mark=0x2,ip actions=output:"pf0vf1_r"
+ cookie=0xdeff105, duration=505314.637s, table=1, n_packets=0, n_bytes=0, priority=100,ct_state=+rel+trk,ct_mark=0x2,ip actions=output:"pf0vf1_r"
+ cookie=0xdeff105, duration=505314.637s, table=1, n_packets=0, n_bytes=0, priority=13,udp,in_port=p0,tp_dst=3784 actions=output:"patch-brp0_c-23",output:"pf0vf1_r"
+ cookie=0xdeff105, duration=505314.637s, table=1, n_packets=493602, n_bytes=48384748, priority=10,dl_dst=42:0b:9a:f1:83:b2 actions=output:"pf0vf1_r"
+ cookie=0xdeff105, duration=505314.637s, table=3, n_packets=694, n_bytes=276779, actions=move:NXM_OF_ETH_DST[]->NXM_OF_ETH_SRC[],mod_dl_dst:42:0b:9a:f1:83:b2,output:"pf0vf1_r"
+
+
+```

go-controller/pkg/config/config.go

@@ -452,7 +452,12 @@ type GatewayConfig struct {
 	Mode GatewayMode `gcfg:"mode"`
 	// Interface is the network interface to use for the gateway in "shared" mode
 	Interface string `gcfg:"interface"`
-	// Exgress gateway interface is the optional network interface to use for external gw pods traffic.
+	// GatewayAcceleratedInterface is the optional network interface to use for gateway traffic acceleration.
+	// This is typically a VF or SF device. When specified it would be used as the in_port for Openflow rules
+	// on the external bridge. The Host IP would be on this device.
+	// Should be used mutually exclusive to the `--gateway-interface` flag.
+	GatewayAcceleratedInterface string `gcfg:"gateway-accelerated-interface"`
+	// Egress gateway interface is the optional network interface to use for external gw pods traffic.
 	EgressGWInterface string `gcfg:"egw-interface"`
 	// NextHop is the gateway IP address of Interface; will be autodetected if not given
 	NextHop string `gcfg:"next-hop"`
@@ -1406,6 +1411,13 @@ var OVNGatewayFlags = []cli.Flag{
 			"interface. Only useful with \"init-gateways\"",
 		Destination: &cliConfig.Gateway.Interface,
 	},
+	&cli.StringFlag{
+		Name: "gateway-accelerated-interface",
+		Usage: "The optional network interface to use for gateway traffic acceleration. " +
+			"This is typically a VF or SF device. When specified it would be used as the in_port for Openflow rules " +
+			"on the external bridge. The Host IP would be on this device.",
+		Destination: &cliConfig.Gateway.GatewayAcceleratedInterface,
+	},
 	&cli.StringFlag{
 		Name: "exgw-interface",
 		Usage: "The interface on nodes that will be used for external gw network traffic. " +

go-controller/pkg/node/default_node_network_controller.go

@@ -1180,9 +1180,8 @@ func (nc *DefaultNodeNetworkController) Start(ctx context.Context) error {
 	// Note(adrianc): DPU deployments are expected to support the new shared gateway changes, upgrade flow
 	// is not needed. Future upgrade flows will need to take DPUs into account.
 	if config.OvnKubeNode.Mode != types.NodeModeDPUHost {
-		bridgeName := ""
 		if config.OvnKubeNode.Mode == types.NodeModeFull {
-			bridgeName = nc.Gateway.GetGatewayBridgeIface()
+			bridgeName := nc.Gateway.GetGatewayIface()
 			// Configure route for svc towards shared gw bridge
 			// Have to have the route to bridge for multi-NIC mode, where the default gateway may go to a non-OVS interface
 			if err := configureSvcRouteViaBridge(nc.routeManager, bridgeName); err != nil {

go-controller/pkg/node/gateway.go

@@ -32,6 +32,7 @@ type Gateway interface {
 	Init(<-chan struct{}, *sync.WaitGroup) error
 	Start()
 	GetGatewayBridgeIface() string
+	GetGatewayIface() string
 	SetDefaultGatewayBridgeMAC(addr net.HardwareAddr)
 	SetDefaultPodNetworkAdvertised(bool)
 	Reconcile() error
@@ -456,6 +457,10 @@ func (g *gateway) GetGatewayBridgeIface() string {
 	return g.openflowManager.getDefaultBridgeName()
 }
 
+func (g *gateway) GetGatewayIface() string {
+	return g.openflowManager.defaultBridge.getGatewayIface()
+}
+
 // getMaxFrameLength returns the maximum frame size (ignoring VLAN header) that a gateway can handle
 func getMaxFrameLength() int {
 	return config.Default.MTU + 14
@@ -537,6 +542,8 @@ type bridgeConfiguration struct {
 	nodeName    string
 	bridgeName  string
 	uplinkName  string
+	gwIface     string
+	gwIfaceRep  string
 	ips         []*net.IPNet
 	interfaceID string
 	macAddress  net.HardwareAddr
@@ -546,11 +553,19 @@ type bridgeConfiguration struct {
 	eipMarkIPs  *markIPsCache
 }
 
+func (b *bridgeConfiguration) getGatewayIface() string {
+	// If gwIface is set, then accelerated GW interface is present and we use it. If else use external bridge instead.
+	if b.gwIface != "" {
+		return b.gwIface
+	}
+	return b.bridgeName
+}
+
 // updateInterfaceIPAddresses sets and returns the bridge's current ips
 func (b *bridgeConfiguration) updateInterfaceIPAddresses(node *corev1.Node) ([]*net.IPNet, error) {
 	b.Lock()
 	defer b.Unlock()
-	ifAddrs, err := getNetworkInterfaceIPAddresses(b.bridgeName)
+	ifAddrs, err := getNetworkInterfaceIPAddresses(b.getGatewayIface())
 	if err != nil {
 		return nil, err
 	}
@@ -578,6 +593,11 @@ func (b *bridgeConfiguration) updateInterfaceIPAddresses(node *corev1.Node) ([]*
 
 func bridgeForInterface(intfName, nodeName, physicalNetworkName string, nodeSubnets, gwIPs []*net.IPNet,
 	advertised bool) (*bridgeConfiguration, error) {
+	var intfRep string
+	var err error
+	isGWAcclInterface := false
+	gwIntf := intfName
+
 	defaultNetConfig := &bridgeUDNConfiguration{
 		masqCTMark:  ctMarkOVN,
 		subnets:     config.Default.ClusterSubnets,
@@ -591,15 +611,47 @@ func bridgeForInterface(intfName, nodeName, physicalNetworkName string, nodeSubn
 		eipMarkIPs: newMarkIPsCache(),
 	}
 	res.netConfig[types.DefaultNetworkName].advertised.Store(advertised)
-	gwIntf := intfName
 
-	if bridgeName, _, err := util.RunOVSVsctl("port-to-br", intfName); err == nil {
+	if config.Gateway.GatewayAcceleratedInterface != "" {
+		// Try to get representor for the specified gateway device.
+		// If function succeeds, then it is either a valid switchdev VF or SF, and we can use this accelerated device
+		// for node IP, Host Ofport for Openflow etc.
+		// If failed - error for improper configuration option
+		intfRep, err = getRepresentor(config.Gateway.GatewayAcceleratedInterface)
+		if err != nil {
+			return nil, fmt.Errorf("gateway accelerated interface %s is not valid: %w", config.Gateway.GatewayAcceleratedInterface, err)
+		}
+		isGWAcclInterface = true
+		klog.Infof("For gateway accelerated interface %s representor: %s", config.Gateway.GatewayAcceleratedInterface, intfRep)
+	}
+
+	if isGWAcclInterface {
+		gatewayAcceleratedInterface := config.Gateway.GatewayAcceleratedInterface
+		bridgeName, _, err := util.RunOVSVsctl("port-to-br", intfRep)
+		if err != nil {
+			return nil, fmt.Errorf("failed to find bridge that has port %s: %w", intfRep, err)
+		}
+		link, err := util.GetNetLinkOps().LinkByName(gatewayAcceleratedInterface)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get netdevice link for %s: %w", gatewayAcceleratedInterface, err)
+		}
+		uplinkName, err := util.GetNicName(bridgeName)
+		if err != nil {
+			return nil, fmt.Errorf("failed to find nic name for bridge %s: %w", bridgeName, err)
+		}
+		res.bridgeName = bridgeName
+		res.uplinkName = uplinkName
+		res.gwIfaceRep = intfRep
+		res.gwIface = gatewayAcceleratedInterface
+		res.macAddress = link.Attrs().HardwareAddr
+	} else if bridgeName, _, err := util.RunOVSVsctl("port-to-br", intfName); err == nil {
 		// This is an OVS bridge's internal port
 		uplinkName, err := util.GetNicName(bridgeName)
 		if err != nil {
 			return nil, fmt.Errorf("failed to find nic name for bridge %s: %w", bridgeName, err)
 		}
 		res.bridgeName = bridgeName
+		res.gwIface = bridgeName
 		res.uplinkName = uplinkName
 		gwIntf = bridgeName
 	} else if _, _, err := util.RunOVSVsctl("br-exists", intfName); err != nil {
@@ -610,6 +662,7 @@ func bridgeForInterface(intfName, nodeName, physicalNetworkName string, nodeSubn
 			return nil, fmt.Errorf("nicToBridge failed for %s: %w", intfName, err)
 		}
 		res.bridgeName = bridgeName
+		res.gwIface = bridgeName
 		res.uplinkName = intfName
 		gwIntf = bridgeName
 	} else {
@@ -625,8 +678,8 @@ func bridgeForInterface(intfName, nodeName, physicalNetworkName string, nodeSubn
 			res.uplinkName = uplinkName
 		}
 		res.bridgeName = intfName
+		res.gwIface = intfName
 	}
-	var err error
 	// Now, we get IP addresses for the bridge
 	if len(gwIPs) > 0 {
 		// use gwIPs if provided
@@ -640,9 +693,11 @@ func bridgeForInterface(intfName, nodeName, physicalNetworkName string, nodeSubn
 		}
 	}
 
-	res.macAddress, err = util.GetOVSPortMACAddress(gwIntf)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get MAC address for ovs port %s: %w", gwIntf, err)
+	if !isGWAcclInterface { // We do not have an accelerated device for Gateway interface
+		res.macAddress, err = util.GetOVSPortMACAddress(gwIntf)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get MAC address for ovs port %s: %w", gwIntf, err)
+		}
 	}
 
 	res.interfaceID, err = bridgedGatewayNodeSetup(nodeName, res.bridgeName, physicalNetworkName)
@@ -665,6 +720,14 @@ func bridgeForInterface(intfName, nodeName, physicalNetworkName string, nodeSubn
 			return nil, err
 		}
 	}
-
 	return &res, nil
 }
+
+func getRepresentor(intfName string) (string, error) {
+	deviceID, err := util.GetDeviceIDFromNetdevice(intfName)
+	if err != nil {
+		return "", err
+	}
+
+	return util.GetFunctionRepresentorName(deviceID)
+}

go-controller/pkg/node/gateway_init.go

@@ -603,7 +603,7 @@ func (nc *DefaultNodeNetworkController) updateGatewayMAC(link netlink.Link) erro
 		return nil
 	}
 
-	if nc.Gateway.GetGatewayBridgeIface() != link.Attrs().Name {
+	if nc.Gateway.GetGatewayIface() != link.Attrs().Name {
 		return nil
 	}
 

go-controller/pkg/node/gateway_shared_intf.go

@@ -1440,10 +1440,10 @@ func flowsForDefaultBridge(bridge *bridgeConfiguration, extraIPs []net.IP) ([]st
 				fmt.Sprintf("cookie=%s, priority=200, in_port=%s, udp, udp_dst=%d, "+
 					"actions=NORMAL", defaultOpenFlowCookie, ofPortPhys, config.Default.EncapPort))
 
-			// table0, Geneve packets coming from LOCAL. Skip conntrack and go directly to external
+			// table0, Geneve packets coming from LOCAL/Host OFPort. Skip conntrack and go directly to external
 			dftFlows = append(dftFlows,
 				fmt.Sprintf("cookie=%s, priority=200, in_port=%s, udp, udp_dst=%d, "+
-					"actions=output:%s", defaultOpenFlowCookie, ovsLocalPort, config.Default.EncapPort, ofPortPhys))
+					"actions=output:%s", defaultOpenFlowCookie, ofPortHost, config.Default.EncapPort, ofPortPhys))
 		}
 		physicalIP, err := util.MatchFirstIPNetFamily(false, bridgeIPs)
 		if err != nil {
@@ -2148,7 +2148,15 @@ func setBridgeOfPorts(bridge *bridgeConfiguration) error {
 				hostRep, stderr, err)
 		}
 	} else {
-		bridge.ofPortHost = ovsLocalPort
+		var err error
+		if bridge.gwIfaceRep != "" {
+			bridge.ofPortHost, _, err = util.RunOVSVsctl("get", "interface", bridge.gwIfaceRep, "ofport")
+			if err != nil {
+				return fmt.Errorf("failed to get ofport of bypass rep %s, error: %v", bridge.gwIfaceRep, err)
+			}
+		} else {
+			bridge.ofPortHost = ovsLocalPort
+		}
 	}
 
 	return nil
@@ -2303,15 +2311,15 @@ func newGateway(
 			// Delete stale masquerade resources if there are any. This is to make sure that there
 			// are no Linux resources with IP from old masquerade subnet when masquerade subnet
 			// gets changed as part of day2 operation.
-			if err := deleteStaleMasqueradeResources(gwBridge.bridgeName, nodeName, watchFactory); err != nil {
+			if err := deleteStaleMasqueradeResources(gwBridge.getGatewayIface(), nodeName, watchFactory); err != nil {
 				return fmt.Errorf("failed to remove stale masquerade resources: %w", err)
 			}
 
-			if err := setNodeMasqueradeIPOnExtBridge(gwBridge.bridgeName); err != nil {
-				return fmt.Errorf("failed to set the node masquerade IP on the ext bridge %s: %v", gwBridge.bridgeName, err)
+			if err := setNodeMasqueradeIPOnExtBridge(gwBridge.getGatewayIface()); err != nil {
+				return fmt.Errorf("failed to set the node masquerade IP on the ext bridge %s: %v", gwBridge.getGatewayIface(), err)
 			}
 
-			if err := addMasqueradeRoute(routeManager, gwBridge.bridgeName, nodeName, gwIPs, watchFactory); err != nil {
+			if err := addMasqueradeRoute(routeManager, gwBridge.getGatewayIface(), nodeName, gwIPs, watchFactory); err != nil {
 				return fmt.Errorf("failed to set the node masquerade route to OVN: %v", err)
 			}
 
@@ -2364,7 +2372,7 @@ func newGateway(
 			gw.openflowManager.requestFlowSync()
 		}
 
-		if err := addHostMACBindings(gwBridge.bridgeName); err != nil {
+		if err := addHostMACBindings(gwBridge.getGatewayIface()); err != nil {
 			return fmt.Errorf("failed to add MAC bindings for service routing: %w", err)
 		}
 
@@ -2421,11 +2429,11 @@ func newNodePortWatcher(
 	subnets = append(subnets, config.Kubernetes.ServiceCIDRs...)
 	if config.Gateway.DisableForwarding {
 		if err := initExternalBridgeServiceForwardingRules(subnets); err != nil {
-			return nil, fmt.Errorf("failed to add accept rules in forwarding table for bridge %s: err %v", gwBridge.bridgeName, err)
+			return nil, fmt.Errorf("failed to add accept rules in forwarding table for bridge %s: err %v", gwBridge.getGatewayIface(), err)
 		}
 	} else {
 		if err := delExternalBridgeServiceForwardingRules(subnets); err != nil {
-			return nil, fmt.Errorf("failed to delete accept rules in forwarding table for bridge %s: err %v", gwBridge.bridgeName, err)
+			return nil, fmt.Errorf("failed to delete accept rules in forwarding table for bridge %s: err %v", gwBridge.getGatewayIface(), err)
 		}
 	}
 
@@ -2443,7 +2451,7 @@ func newNodePortWatcher(
 		gatewayIPv4:    gatewayIPv4,
 		gatewayIPv6:    gatewayIPv6,
 		ofportPhys:     ofportPhys,
-		gwBridge:       gwBridge.bridgeName,
+		gwBridge:       gwBridge.getGatewayIface(),
 		serviceInfo:    make(map[ktypes.NamespacedName]*serviceConfig),
 		nodeIPManager:  nodeIPManager,
 		ofm:            ofm,