feat: Replicate host routing rules and tables in pod

This change enables DRANET to discover and replicate host routing rules and
routes from non-default tables into the pod's network namespace.

Previously, only routes from the main table were copied. This was insufficient
for advanced networking configurations that rely on policy-based routing (ip
rule) and multiple routing tables (like source based routing)

This commit introduces the following changes:
- The driver now inspects the host for ip rule entries associated with a network
  interface.
- It also inspects for ip route entries in non-default tables.
- These rules and routes are then replicated within the pod's network namespace
  when it's created.
- An end-to-end test has been added to validate this functionality.

Author: gauravkghildiyal

pkg/apis/types.go

@@ -26,6 +26,9 @@ type NetworkConfig struct {
 	// Routes defines static routes to be configured for this interface.
 	Routes []RouteConfig `json:"routes,omitempty"`
 
+	// Rules defines routing rules to be configured for this interface.
+	Rules []RuleConfig `json:"rules,omitempty"`
+
 	// Neighbors defines permanent neighbor (ARP/NDP) entries to be added for this interface.
 	Neighbors []NeighborConfig `json:"neighbors,omitempty"`
 
@@ -86,6 +89,20 @@ type RouteConfig struct {
 	// Scope is the scope of the route (e.g., link, host, global).
 	// Refers to Linux route scopes (e.g., 0 for RT_SCOPE_UNIVERSE, 253 for RT_SCOPE_LINK).
 	Scope uint8 `json:"scope,omitempty"`
+	// Table is the routing table to use for the route.
+	Table int `json:"table,omitempty"`
+}
+
+// RuleConfig represents a network rule configuration.
+type RuleConfig struct {
+	// Priority is the priority of the rule.
+	Priority int `json:"priority,omitempty"`
+	// Source is the source IP address for the rule.
+	Source string `json:"source,omitempty"`
+	// Destination is the destination IP address for the rule.
+	Destination string `json:"destination,omitempty"`
+	// Table is the routing table to use for the rule.
+	Table int `json:"table,omitempty"`
 }
 
 // NeighborConfig represents a neighbor (ARP/NDP) entry.

pkg/apis/validation.go

@@ -67,6 +67,11 @@ func ValidateConfig(raw *runtime.RawExtension) (*NetworkConfig, []error) {
 		allErrors = append(allErrors, validateRoutes(config.Routes, "routes")...)
 	}
 
+	// Validate Rules
+	if len(config.Rules) > 0 {
+		allErrors = append(allErrors, validateRules(config.Rules, "rules")...)
+	}
+
 	// Validate EthtoolConfig if present
 	if config.Ethtool != nil {
 		allErrors = append(allErrors, validateEthtoolConfig(config.Ethtool, "ethtool")...)
@@ -204,6 +209,38 @@ func validateRoutes(routes []RouteConfig, fieldPath string) (allErrors []error)
 				allErrors = append(allErrors, fmt.Errorf("%s.source: invalid IP address format '%s'", currentFieldPath, route.Source))
 			}
 		}
+
+		if route.Table < 0 {
+			allErrors = append(allErrors, fmt.Errorf("%s.table: must be a non-negative integer, got %d", currentFieldPath, route.Table))
+		}
+	}
+	return allErrors
+}
+
+// validateRules validates a slice of RuleConfig.
+func validateRules(rules []RuleConfig, fieldPath string) (allErrors []error) {
+	for i, rule := range rules {
+		currentFieldPath := fmt.Sprintf("%s[%d]", fieldPath, i)
+
+		if rule.Priority < 0 || rule.Priority > 32767 {
+			allErrors = append(allErrors, fmt.Errorf("%s.priority: must be an integer between 0 and 32767, got %d", currentFieldPath, rule.Priority))
+		}
+
+		if rule.Table < 0 {
+			allErrors = append(allErrors, fmt.Errorf("%s.table: must be a non-negative integer, got %d", currentFieldPath, rule.Table))
+		}
+
+		if rule.Source != "" {
+			if _, _, err := net.ParseCIDR(rule.Source); err != nil {
+				allErrors = append(allErrors, fmt.Errorf("%s.source: invalid CIDR format '%s'", currentFieldPath, rule.Source))
+			}
+		}
+
+		if rule.Destination != "" {
+			if _, _, err := net.ParseCIDR(rule.Destination); err != nil {
+				allErrors = append(allErrors, fmt.Errorf("%s.destination: invalid CIDR format '%s'", currentFieldPath, rule.Destination))
+			}
+		}
 	}
 	return allErrors
 }

pkg/apis/validation_test.go

@@ -48,10 +48,14 @@ func TestValidateConfig(t *testing.T) {
 		Routes: []RouteConfig{
 			{Destination: "0.0.0.0/0", Gateway: "192.168.1.254", Scope: unix.RT_SCOPE_UNIVERSE},
 		},
+		Rules: []RuleConfig{
+			{Source: "10.0.0.0/8", Table: 100},
+		},
 		Ethtool: &EthtoolConfig{Features: map[string]bool{"tso": true}},
 	}
 	invalidInterfaceConf := NetworkConfig{Interface: InterfaceConfig{Name: "eth/0"}}
 	invalidRouteConf := NetworkConfig{Interface: InterfaceConfig{Name: "eth0"}, Routes: []RouteConfig{{Destination: "invalid-cidr"}}}
+	invalidRuleConf := NetworkConfig{Interface: InterfaceConfig{Name: "eth0"}, Rules: []RuleConfig{{Source: "invalid-cidr"}}}
 
 	tests := []struct {
 		name        string
@@ -106,6 +110,13 @@ func TestValidateConfig(t *testing.T) {
 			expectedCfg: &invalidRouteConf,
 			errContains: []string{"routes[0].destination: invalid IP or CIDR format 'invalid-cidr'"},
 		},
+		{
+			name:        "config with rule validation error",
+			raw:         newRawExtension(t, invalidRuleConf),
+			expectErr:   true,
+			expectedCfg: &invalidRuleConf,
+			errContains: []string{"rules[0].source: invalid CIDR format 'invalid-cidr'"},
+		},
 	}
 
 	for _, tt := range tests {
@@ -323,6 +334,19 @@ func TestValidateRoutes(t *testing.T) {
 			fieldPath: "routes",
 			expectErr: false,
 		},
+		{
+			name:      "valid route with table",
+			routes:    []RouteConfig{{Destination: "10.10.10.0/24", Gateway: "192.168.1.1", Table: 100}},
+			fieldPath: "routes",
+			expectErr: false,
+		},
+		{
+			name:      "invalid route with negative table",
+			routes:    []RouteConfig{{Destination: "10.10.10.0/24", Gateway: "192.168.1.1", Table: -1}},
+			fieldPath: "routes",
+			expectErr: true,
+			errCount:  1,
+		},
 		{
 			name:      "empty destination",
 			routes:    []RouteConfig{{Gateway: "192.168.1.1"}},
@@ -387,6 +411,89 @@ func TestValidateRoutes(t *testing.T) {
 	}
 }
 
+func TestValidateRules(t *testing.T) {
+	tests := []struct {
+		name      string
+		rules     []RuleConfig
+		fieldPath string
+		expectErr bool
+		errCount  int
+	}{
+		{
+			name:      "valid rule",
+			rules:     []RuleConfig{{Source: "10.0.0.0/8", Table: 100, Priority: 10}},
+			fieldPath: "rules",
+			expectErr: false,
+		},
+		{
+			name:      "valid rule - priority at min",
+			rules:     []RuleConfig{{Priority: 0, Table: 100}},
+			fieldPath: "rules",
+			expectErr: false,
+		},
+		{
+			name:      "valid rule - priority at max",
+			rules:     []RuleConfig{{Priority: 32767, Table: 100}},
+			fieldPath: "rules",
+			expectErr: false,
+		},
+		{
+			name:      "invalid priority - too high",
+			rules:     []RuleConfig{{Priority: 32768}},
+			fieldPath: "rules",
+			expectErr: true,
+			errCount:  1,
+		},
+		{
+			name:      "invalid priority - negative",
+			rules:     []RuleConfig{{Priority: -1}},
+			fieldPath: "rules",
+			expectErr: true,
+			errCount:  1,
+		},
+		{
+			name:      "invalid table",
+			rules:     []RuleConfig{{Table: -1}},
+			fieldPath: "rules",
+			expectErr: true,
+			errCount:  1,
+		},
+		{
+			name:      "invalid source CIDR",
+			rules:     []RuleConfig{{Source: "invalid-cidr"}},
+			fieldPath: "rules",
+			expectErr: true,
+			errCount:  1,
+		},
+		{
+			name:      "invalid destination CIDR",
+			rules:     []RuleConfig{{Destination: "invalid-cidr"}},
+			fieldPath: "rules",
+			expectErr: true,
+			errCount:  1,
+		},
+		{
+			name:      "multiple errors",
+			rules:     []RuleConfig{{Priority: -1, Table: -1, Source: "invalid", Destination: "invalid"}},
+			fieldPath: "rules",
+			expectErr: true,
+			errCount:  4,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			errs := validateRules(tt.rules, tt.fieldPath)
+			if (len(errs) > 0) != tt.expectErr {
+				t.Errorf("validateRules() expectErr %v, got errors: %v", tt.expectErr, errs)
+			}
+			if tt.expectErr && len(errs) != tt.errCount {
+				t.Errorf("validateRules() expected %d errors, got %d: %v", tt.errCount, len(errs), errs)
+			}
+		})
+	}
+}
+
 func TestValidateNeighborConfig(t *testing.T) {
 	tests := []struct {
 		name      string

pkg/driver/dra_hooks.go

@@ -134,6 +134,8 @@ func (np *NetworkDriver) prepareResourceClaims(ctx context.Context, claims []*re
 // prepareResourceClaim gets all the configuration required to be applied at runtime and passes it downs to the handlers.
 // This happens in the kubelet so it can be a "slow" operation, so we can execute fast in RunPodsandbox, that happens in the
 // container runtime and has strong expectactions to be executed fast (default hook timeout is 2 seconds).
+//
+// TODO(#290): This function has grown too large and needs to be split apart.
 func (np *NetworkDriver) prepareResourceClaim(ctx context.Context, claim *resourceapi.ResourceClaim) kubeletplugin.PrepareResult {
 	klog.V(2).Infof("PrepareResourceClaim Claim %s/%s", claim.Namespace, claim.Name)
 	start := time.Now()
@@ -161,6 +163,13 @@ func (np *NetworkDriver) prepareResourceClaim(ctx context.Context, claim *resour
 		}
 	}
 
+	rulesByTable, err := getRuleInfo(nlHandle)
+	if err != nil {
+		return kubeletplugin.PrepareResult{
+			Err: fmt.Errorf("error getting rule info: %v", err),
+		}
+	}
+
 	var errorList []error
 	charDevices := sets.New[string]()
 	for _, result := range claim.Status.Allocation.Devices.Results {
@@ -284,35 +293,21 @@ func (np *NetworkDriver) prepareResourceClaim(ctx context.Context, claim *resour
 			podCfg.NetworkInterfaceConfigInPod.Ethtool.Features = ethtoolFeatures
 		}
 
-		// Obtain the routes associated to the interface
-		// TODO: only considers outgoing traffic
-		filter := &netlink.Route{
-			LinkIndex: link.Attrs().Index,
-		}
-		routes, err := nlHandle.RouteListFiltered(netlink.FAMILY_ALL, filter, netlink.RT_FILTER_OIF)
+		// Obtain the routes and rules associated with the interface.
+		routes, tables, err := getRouteInfo(nlHandle, ifName, link)
 		if err != nil {
-			klog.Infof("fail to get ip routes for interface %s : %v", ifName, err)
+			errorList = append(errorList, err)
+			continue
 		}
-		for _, route := range routes {
-			routeCfg := apis.RouteConfig{}
-			// routes need a destination
-			if route.Dst == nil {
-				continue
-			}
-			// Discard IPv6 link-local routes, but allow IPv4 link-local.
-			if route.Dst.IP.To4() == nil && route.Dst.IP.IsLinkLocalUnicast() {
-				continue
+		podCfg.NetworkInterfaceConfigInPod.Routes = append(podCfg.NetworkInterfaceConfigInPod.Routes, routes...)
+
+		for _, table := range tables.UnsortedList() {
+			if rules, ok := rulesByTable[table]; ok {
+				klog.V(5).Infof("Adding %d rules for table %d associated with interface %s", len(rules), table, ifName)
+				podCfg.NetworkInterfaceConfigInPod.Rules = append(podCfg.NetworkInterfaceConfigInPod.Rules, rules...)
+				// Avoid adding the same rule twice
+				delete(rulesByTable, table)
 			}
-			routeCfg.Destination = route.Dst.String()
-
-			if route.Gw != nil {
-				routeCfg.Gateway = route.Gw.String()
-			}
-			if route.Src != nil {
-				routeCfg.Source = route.Src.String()
-			}
-			routeCfg.Scope = uint8(route.Scope)
-			podCfg.NetworkInterfaceConfigInPod.Routes = append(podCfg.NetworkInterfaceConfigInPod.Routes, routeCfg)
 		}
 
 		// Obtain the neighbors associated to the interface
@@ -446,3 +441,85 @@ func formatDeviceNames(devices []resourceapi.Device, max int) string {
 
 	return fmt.Sprintf("%s, and %d more", strings.Join(deviceNames[:max], ", "), len(deviceNames)-max)
 }
+
+// getRuleInfo lists all IP rules in the host network namespace and groups them
+// by the route table they are associated with. It returns a map where keys are
+// table IDs and values are slices of RuleConfig. Rules associated with the
+// main or local tables are ignored.
+func getRuleInfo(nlHandle nlwrap.Handle) (map[int][]apis.RuleConfig, error) {
+	rulesByTable := make(map[int][]apis.RuleConfig)
+	rules, err := nlHandle.RuleList(netlink.FAMILY_ALL)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get ip rules: %w", err)
+	}
+	for _, rule := range rules {
+		ruleCfg := apis.RuleConfig{
+			Priority: rule.Priority,
+			Table:    rule.Table,
+		}
+		if rule.Src != nil {
+			ruleCfg.Source = rule.Src.String()
+		}
+		if rule.Dst != nil {
+			ruleCfg.Destination = rule.Dst.String()
+		}
+		// Only care about rules with route tables associated, and exclude main and local tables.
+		if rule.Table > 0 && rule.Table != unix.RT_TABLE_MAIN && rule.Table != unix.RT_TABLE_LOCAL {
+			klog.V(5).Infof("Found rule %s for table %d", rule.String(), rule.Table)
+			rulesByTable[rule.Table] = append(rulesByTable[rule.Table], ruleCfg)
+		}
+	}
+	return rulesByTable, nil
+}
+
+// getRouteInfo retrieves all routes associated with a given network interface.
+// It filters out routes that are not suitable for pod namespaces, such as
+// routes in the local table. It returns the list of suitable routes and a set
+// of the route table IDs to which they belong.
+func getRouteInfo(nlHandle nlwrap.Handle, ifName string, link netlink.Link) ([]apis.RouteConfig, sets.Set[int], error) {
+	routes := []apis.RouteConfig{}
+	tables := sets.Set[int]{}
+	filter := &netlink.Route{
+		LinkIndex: link.Attrs().Index,
+	}
+	rl, err := nlHandle.RouteListFiltered(netlink.FAMILY_ALL, filter, netlink.RT_FILTER_OIF|netlink.RT_FILTER_TABLE)
+	if err != nil {
+		return nil, nil, fmt.Errorf("fail to get ip routes for interface %s : %w", ifName, err)
+	}
+	for _, route := range rl {
+		routeCfg := apis.RouteConfig{}
+		// routes need a destination
+		if route.Dst == nil {
+			klog.V(5).Infof("Skipping route %s for interface %s because it has no destination", route.String(), ifName)
+			continue
+		}
+		// Do not copy routes from the local table because they are specific
+		// to the host and the kernel will manage the local routing
+		// table within the pod's network namespace.
+		if route.Table == unix.RT_TABLE_LOCAL {
+			klog.V(5).Infof("Skipping route %s for interface %s because it is in the local table", route.String(), ifName)
+			continue
+		}
+		// Discard IPv6 link-local routes, but allow IPv4 link-local.
+		if route.Dst.IP.To4() == nil && route.Dst.IP.IsLinkLocalUnicast() {
+			klog.V(5).Infof("Skipping IPv6 link-local route %s for interface %s", route.String(), ifName)
+			continue
+		}
+		routeCfg.Destination = route.Dst.String()
+		if route.Gw != nil {
+			routeCfg.Gateway = route.Gw.String()
+		}
+		if route.Src != nil {
+			routeCfg.Source = route.Src.String()
+		}
+		routeCfg.Scope = uint8(route.Scope)
+		routeCfg.Table = route.Table
+		routes = append(routes, routeCfg)
+		// Collect table IDs for rules lookup later.
+		if route.Table > 0 {
+			klog.V(5).Infof("Found route table %d for interface %s", route.Table, ifName)
+			tables.Insert(route.Table)
+		}
+	}
+	return routes, tables, nil
+}