Add an unreachable default route for a UDN

This commit is to add a default unreachable route while creating
an user defined network:

unreachable default metric 4278198272

This PR also allows to mention route priority and type while adding
a route to a VRF.

Signed-off-by: Arnab Ghosh <[email protected]>

Author: arghosh93

go-controller/pkg/node/controllers/egressip/egressip_test.go

@@ -19,6 +19,7 @@ import (
 	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
 	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -1133,7 +1134,7 @@ var _ = ginkgo.Describe("VRF", func() {
 		ginkgo.By("create VRF and add link")
 		gomega.Expect(createVRFAndEnslaveLink(testNS, dummyLink1Name, vrfName, vrfTable)).Should(gomega.Succeed())
 		ginkgo.By("add route to routing table associated with VRF")
-		vrfRoute := getDstRouteForTable(getLinkIndex(dummyLink1Name), int(vrfTable), dummy3IPv4CIDR)
+		vrfRoute := getV4DstRouteForTable(getLinkIndex(dummyLink1Name), int(vrfTable), dummy3IPv4CIDR)
 		gomega.Expect(testNS.Do(func(ns.NetNS) error {
 			return netlink.RouteAdd(&vrfRoute)
 		})).Should(gomega.Succeed())
@@ -1642,31 +1643,49 @@ func isIPTableRuleArgIPV6(ruleArgs []string) bool {
 	panic("unable to determine IP version of IPTable rule")
 }
 
+// For any IPV4 or IPV6 route, default type is unicast if not mentioned explicitly
+// Ref: https://www.man7.org/linux/man-pages/man8/ip-route.8.html
 func getDefaultIPv4Route(linkIndex int) netlink.Route {
 	// dst is nil because netlink represents a default route as nil
-	return netlink.Route{LinkIndex: linkIndex, Table: util.CalculateRouteTableID(linkIndex), Dst: defaultV4AnyCIDR}
+	return netlink.Route{LinkIndex: linkIndex, Table: util.CalculateRouteTableID(linkIndex), Dst: defaultV4AnyCIDR, Type: unix.RTN_UNICAST}
 }
 
 func getDefaultIPv6Route(linkIndex int) netlink.Route {
 	// dst is nil because netlink represents a default route as nil
-	return netlink.Route{LinkIndex: linkIndex, Table: util.CalculateRouteTableID(linkIndex), Dst: defaultV6AnyCIDR}
+	// Priority for a default IPv6 route gets set to 1024
+	// Ref : https://access.redhat.com/solutions/3659171
+	return netlink.Route{LinkIndex: linkIndex, Table: util.CalculateRouteTableID(linkIndex), Dst: defaultV6AnyCIDR, Type: unix.RTN_UNICAST, Priority: 1024}
 }
 
 func getDstRoute(linkIndex int, dst string) netlink.Route {
-	return getDstRouteForTable(linkIndex, util.CalculateRouteTableID(linkIndex), dst)
+	if utilnet.IsIPv6CIDRString(dst) {
+		return getV6DstRouteForTable(linkIndex, util.CalculateRouteTableID(linkIndex), dst)
+	} else {
+		return getV4DstRouteForTable(linkIndex, util.CalculateRouteTableID(linkIndex), dst)
+	}
+
+}
 
+// Default scope is link for direct unicast routes
+// Ref: https://www.man7.org/linux/man-pages/man8/ip-route.8.html
+func getV4DstRouteForTable(linkIndex, table int, dst string) netlink.Route {
+	_, dstIPNet, err := net.ParseCIDR(dst)
+	if err != nil {
+		panic(err.Error())
+	}
+	return netlink.Route{LinkIndex: linkIndex, Dst: dstIPNet, Table: table, Type: unix.RTN_UNICAST, Scope: netlink.SCOPE_LINK}
 }
 
-func getDstRouteForTable(linkIndex, table int, dst string) netlink.Route {
+func getV6DstRouteForTable(linkIndex, table int, dst string) netlink.Route {
 	_, dstIPNet, err := net.ParseCIDR(dst)
 	if err != nil {
 		panic(err.Error())
 	}
-	return netlink.Route{LinkIndex: linkIndex, Dst: dstIPNet, Table: table}
+	return netlink.Route{LinkIndex: linkIndex, Dst: dstIPNet, Table: table, Type: unix.RTN_UNICAST, Priority: 256}
 }
 
 func getLinkLocalRoute(linkIndex int) netlink.Route {
-	return netlink.Route{LinkIndex: linkIndex, Dst: linkLocalCIDR, Table: util.CalculateRouteTableID(linkIndex)}
+	return netlink.Route{LinkIndex: linkIndex, Dst: linkLocalCIDR, Table: util.CalculateRouteTableID(linkIndex), Type: unix.RTN_UNICAST, Priority: 256}
 }
 
 func getNetlinkAddr(ip, netmask string) *netlink.Addr {

go-controller/pkg/node/gateway_udn.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -670,7 +671,31 @@ func (udng *UserDefinedNetworkGateway) computeRoutesForUDN(mpLink netlink.Link)
 			}
 		}
 	}
-
+	// Add unreachable route to enure that kernel always finds a match to the VRF table rather than
+	// referring to default VRF table and send traffic via unwanted interfaces and to unwanted gateway.
+	// non 0 link index for an unreachable or blackhole IPv4 route returns 'invalid argument'
+	hasV4Subnet, hasV6Subnet := udng.IPMode()
+	if hasV4Subnet {
+		_, v4AnyCIDR, _ := net.ParseCIDR("0.0.0.0/0")
+		retVal = append(retVal, netlink.Route{
+			Dst:      v4AnyCIDR,
+			Table:    udng.vrfTableId,
+			Priority: 4278198272,
+			Type:     unix.RTN_UNREACHABLE,
+		})
+	}
+	// link index for an unreachable IPv6 route always get set to 1. Link index 1 refers to default loopback
+	// device at all time. Reference: https://docs.kernel.org/networking/vrf.html#using-iproute2-for-vrfs
+	if hasV6Subnet {
+		_, v6AnyCIDR, _ := net.ParseCIDR("::/0")
+		retVal = append(retVal, netlink.Route{
+			LinkIndex: types.LoopbackInterfaceIndex,
+			Dst:       v6AnyCIDR,
+			Table:     udng.vrfTableId,
+			Priority:  4278198272,
+			Type:      unix.RTN_UNREACHABLE,
+		})
+	}
 	return retVal, nil
 }
 

go-controller/pkg/node/gateway_udn_test.go

@@ -14,6 +14,7 @@ import (
 	nadfake "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/fake"
 	"github.com/stretchr/testify/mock"
 	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -1051,7 +1052,7 @@ var _ = Describe("UserDefinedNetworkGateway", func() {
 
 			routes, err := udnGateway.computeRoutesForUDN(mplink)
 			Expect(err).NotTo(HaveOccurred())
-			Expect(routes).To(HaveLen(7))
+			Expect(routes).To(HaveLen(9))
 			Expect(err).NotTo(HaveOccurred())
 			Expect(*routes[0].Dst).To(Equal(*ovntest.MustParseIPNet("172.16.1.0/24"))) // default service subnet
 			Expect(routes[0].LinkIndex).To(Equal(bridgelink.Attrs().Index))
@@ -1089,7 +1090,7 @@ var _ = Describe("UserDefinedNetworkGateway", func() {
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fexec.CalledMatchesExpected()).To(BeTrue(), fexec.ErrorDesc)
 	})
-	ovntest.OnSupportedPlatformsIt("should compute correct service routes for a user defined network", func() {
+	ovntest.OnSupportedPlatformsIt("should compute correct routes for a user defined network", func() {
 		config.Gateway.Interface = "eth0"
 		config.IPv4Mode = true
 		config.IPv6Mode = true
@@ -1122,7 +1123,7 @@ var _ = Describe("UserDefinedNetworkGateway", func() {
 
 			routes, err := udnGateway.computeRoutesForUDN(mplink)
 			Expect(err).NotTo(HaveOccurred())
-			Expect(routes).To(HaveLen(8))
+			Expect(routes).To(HaveLen(10))
 			Expect(err).NotTo(HaveOccurred())
 			// 1st and 2nd routes are the service routes from user-provided config value
 			Expect(*routes[0].Dst).To(Equal(*config.Kubernetes.ServiceCIDRs[0]))
@@ -1157,6 +1158,16 @@ var _ = Describe("UserDefinedNetworkGateway", func() {
 			Expect(*routes[7].Dst).To(Equal(*ovntest.MustParseIPNet("ae70::/60"))) // cluster subnet route
 			Expect(routes[7].LinkIndex).To(Equal(mplink.Attrs().Index))
 			Expect(routes[7].Gw.Equal(ovntest.MustParseIP("ae70::1"))).To(BeTrue())
+
+			// IPv4 default unreachable route
+			Expect(*routes[8].Dst).To(Equal(*ovntest.MustParseIPNet("0.0.0.0/0"))) // cluster subnet route
+			Expect(routes[8].Priority).To(Equal(4278198272))
+			Expect(routes[8].Type).To(Equal(unix.RTN_UNREACHABLE))
+
+			// IPv6 default unreachable route
+			Expect(*routes[9].Dst).To(Equal(*ovntest.MustParseIPNet("::/0"))) // cluster subnet route
+			Expect(routes[9].Priority).To(Equal(4278198272))
+			Expect(routes[9].Type).To(Equal(unix.RTN_UNREACHABLE))
 			return nil
 		})
 		Expect(err).NotTo(HaveOccurred())

go-controller/pkg/node/routemanager/route_manager.go

@@ -101,11 +101,13 @@ func (c *Controller) addRoute(r netlink.Route) error {
 		// already managed - nothing to do
 		return nil
 	}
-	link, err := util.GetNetLinkOps().LinkByIndex(r.LinkIndex)
-	if err != nil {
-		return fmt.Errorf("failed to apply route (%s) because unable to get link: %v", r.String(), err)
+	if r.LinkIndex != 0 {
+		_, err := util.GetNetLinkOps().LinkByIndex(r.LinkIndex)
+		if err != nil {
+			return fmt.Errorf("failed to apply route (%s) because unable to get link: %v", r.String(), err)
+		}
 	}
-	if err := c.applyRoute(link, r.Gw, r.Dst, r.MTU, r.Src, r.Table); err != nil {
+	if err := c.applyRoute(r.LinkIndex, r.Gw, r.Dst, r.MTU, r.Src, r.Table, r.Priority, r.Type, r.Scope); err != nil {
 		return fmt.Errorf("failed to apply route (%s): %v", r.String(), err)
 	}
 	klog.Infof("Route Manager: completed adding route: %s", r.String())
@@ -118,15 +120,17 @@ func (c *Controller) delRoute(r netlink.Route) error {
 	c.Lock()
 	defer c.Unlock()
 	klog.Infof("Route Manager: attempting to delete route: %s", r.String())
-	link, err := util.GetNetLinkOps().LinkByIndex(r.LinkIndex)
-	if err != nil {
-		if util.GetNetLinkOps().IsLinkNotFoundError(err) {
-			delete(c.store, r.LinkIndex)
-			return nil
+	if r.LinkIndex != 0 {
+		_, err := util.GetNetLinkOps().LinkByIndex(r.LinkIndex)
+		if err != nil {
+			if util.GetNetLinkOps().IsLinkNotFoundError(err) {
+				delete(c.store, r.LinkIndex)
+				return nil
+			}
+			return fmt.Errorf("failed to delete route (%s) because unable to get link: %v", r.String(), err)
 		}
-		return fmt.Errorf("failed to delete route (%s) because unable to get link: %v", r.String(), err)
 	}
-	if err := c.netlinkDelRoute(link, r.Dst, r.Table); err != nil {
+	if err := c.netlinkDelRoute(r.LinkIndex, r.Dst, r.Table); err != nil {
 		return fmt.Errorf("failed to delete route (%s): %v", r.String(), err)
 	}
 	managedRoutes, ok := c.store[r.LinkIndex]
@@ -171,27 +175,31 @@ func (c *Controller) processNetlinkEvent(ru netlink.RouteUpdate) error {
 	}
 	for _, managedRoute := range managedRoutes {
 		if RoutePartiallyEqual(managedRoute, ru.Route) {
-			link, err := util.GetNetLinkOps().LinkByIndex(managedRoute.LinkIndex)
-			if err != nil {
-				klog.Errorf("Route Manager: failed to restore route because unable to get link by index %d: %v", managedRoute.LinkIndex, err)
-				continue
+			if managedRoute.LinkIndex != 0 {
+				_, err := util.GetNetLinkOps().LinkByIndex(managedRoute.LinkIndex)
+				if err != nil {
+					klog.Errorf("Route Manager: failed to restore route because unable to get link by index %d: %v", managedRoute.LinkIndex, err)
+					continue
+				}
 			}
-			if err = c.applyRoute(link, managedRoute.Gw, managedRoute.Dst, managedRoute.MTU, managedRoute.Src, managedRoute.Table); err != nil {
+			if err := c.applyRoute(managedRoute.LinkIndex, managedRoute.Gw, managedRoute.Dst, managedRoute.MTU, managedRoute.Src, managedRoute.Table,
+				managedRoute.Priority, managedRoute.Type, managedRoute.Scope); err != nil {
 				klog.Errorf("Route Manager: failed to apply route (%s): %v", managedRoute.String(), err)
 			}
 		}
 	}
 	return nil
 }
 
-func (c *Controller) applyRoute(link netlink.Link, gwIP net.IP, subnet *net.IPNet, mtu int, src net.IP, table int) error {
-	filterRoute, filterMask := filterRouteByDstAndTable(link.Attrs().Index, subnet, table)
+func (c *Controller) applyRoute(linkIndex int, gwIP net.IP, subnet *net.IPNet, mtu int, src net.IP,
+	table, priority, rtype int, scope netlink.Scope) error {
+	filterRoute, filterMask := filterRouteByDstAndTable(linkIndex, subnet, table)
 	existingRoutes, err := util.GetNetLinkOps().RouteListFiltered(getNetlinkIPFamily(subnet), filterRoute, filterMask)
 	if err != nil {
 		return fmt.Errorf("failed to list filtered routes: %v", err)
 	}
 	if len(existingRoutes) == 0 {
-		return c.netlinkAddRoute(link, gwIP, subnet, mtu, src, table)
+		return c.netlinkAddRoute(linkIndex, gwIP, subnet, mtu, src, table, priority, rtype, scope)
 	}
 	netlinkRoute := &existingRoutes[0]
 	if netlinkRoute.MTU != mtu || !src.Equal(netlinkRoute.Src) || !gwIP.Equal(netlinkRoute.Gw) {
@@ -207,10 +215,11 @@ func (c *Controller) applyRoute(link netlink.Link, gwIP net.IP, subnet *net.IPNe
 	return nil
 }
 
-func (c *Controller) netlinkAddRoute(link netlink.Link, gwIP net.IP, subnet *net.IPNet, mtu int, srcIP net.IP, table int) error {
+func (c *Controller) netlinkAddRoute(linkIndex int, gwIP net.IP, subnet *net.IPNet,
+	mtu int, srcIP net.IP, table, priority, rtype int, scope netlink.Scope) error {
 	newNlRoute := &netlink.Route{
 		Dst:       subnet,
-		LinkIndex: link.Attrs().Index,
+		LinkIndex: linkIndex,
 		Scope:     netlink.SCOPE_UNIVERSE,
 		Table:     table,
 	}
@@ -223,21 +232,31 @@ func (c *Controller) netlinkAddRoute(link netlink.Link, gwIP net.IP, subnet *net
 	if mtu != 0 {
 		newNlRoute.MTU = mtu
 	}
+	if priority != 0 {
+		newNlRoute.Priority = priority
+	}
+	if rtype != 0 {
+		newNlRoute.Type = rtype
+	}
+	if scope != netlink.Scope(0) {
+		newNlRoute.Scope = scope
+	}
 	err := util.GetNetLinkOps().RouteAdd(newNlRoute)
 	if err != nil {
-		return fmt.Errorf("failed to add route (gw: %v, subnet %v, mtu %d, src IP %v): %v", gwIP, subnet, mtu, srcIP, err)
+		return fmt.Errorf("failed to add route (linkIndex: %d gw: %v, subnet %v, mtu %d, src IP %v): %v",
+			newNlRoute.LinkIndex, gwIP, subnet, mtu, srcIP, err)
 	}
 	return nil
 }
 
-func (c *Controller) netlinkDelRoute(link netlink.Link, subnet *net.IPNet, table int) error {
+func (c *Controller) netlinkDelRoute(linkIndex int, subnet *net.IPNet, table int) error {
 	if subnet == nil {
 		return fmt.Errorf("cannot delete route with no valid subnet")
 	}
-	filter, mask := filterRouteByDstAndTable(link.Attrs().Index, subnet, table)
+	filter, mask := filterRouteByDstAndTable(linkIndex, subnet, table)
 	existingRoutes, err := util.GetNetLinkOps().RouteListFiltered(netlink.FAMILY_ALL, filter, mask)
 	if err != nil {
-		return fmt.Errorf("failed to get routes for link %s: %v", link.Attrs().Name, err)
+		return fmt.Errorf("failed to get routes for link %d: %v", linkIndex, err)
 	}
 	for _, existingRoute := range existingRoutes {
 		if err = util.GetNetLinkOps().RouteDel(&existingRoute); err != nil {
@@ -286,16 +305,19 @@ func (c *Controller) sync() {
 				}
 			}
 			if !found {
-				link, err := util.GetNetLinkOps().LinkByIndex(managedRoute.LinkIndex)
-				if err != nil {
-					if util.GetNetLinkOps().IsLinkNotFoundError(err) {
-						deletedLinkIndexes = append(deletedLinkIndexes, linkIndex)
-					} else {
-						klog.Errorf("Route Manager: failed to apply route (%s) because unable to retrieve associated link: %v", managedRoute.String(), err)
+				if managedRoute.LinkIndex != 0 {
+					_, err := util.GetNetLinkOps().LinkByIndex(managedRoute.LinkIndex)
+					if err != nil {
+						if util.GetNetLinkOps().IsLinkNotFoundError(err) {
+							deletedLinkIndexes = append(deletedLinkIndexes, linkIndex)
+						} else {
+							klog.Errorf("Route Manager: failed to apply route (%s) because unable to retrieve associated link: %v", managedRoute.String(), err)
+						}
+						continue
 					}
-					continue
 				}
-				if err := c.applyRoute(link, managedRoute.Gw, managedRoute.Dst, managedRoute.MTU, managedRoute.Src, managedRoute.Table); err != nil {
+				if err := c.applyRoute(managedRoute.LinkIndex, managedRoute.Gw, managedRoute.Dst, managedRoute.MTU, managedRoute.Src, managedRoute.Table,
+					managedRoute.Priority, managedRoute.Type, managedRoute.Scope); err != nil {
 					klog.Errorf("Route Manager: failed to apply route (%s): %v", managedRoute.String(), err)
 				}
 			}
@@ -353,5 +375,8 @@ func RoutePartiallyEqual(r, x netlink.Route) bool {
 		r.Gw.Equal(x.Gw) &&
 		r.Table == x.Table &&
 		r.Flags == x.Flags &&
-		r.MTU == x.MTU
+		r.MTU == x.MTU &&
+		r.Type == x.Type &&
+		r.Priority == x.Priority &&
+		r.Scope == x.Scope
 }