Configure mgmtport-no-snat-subnets sets elements

Currently traffic gets SNATed at ovn-k8s-mp0 within the mgmtport-snat chain.
Since OVNK has transitioned to nftables, this behavior can
no longer be overridden.

Previously, with iptables, SNAT could be avoided by adding a
higher-priority rule in the POSTROUTING chain. However, with nftables,
all rules are evaluated before making a final decision, making it
impossible to skip SNAT.

Some applications, like Submariner, need to preserve the source IP
when traffic reaches the destination pod, as certain use cases depend on it.

This PR Update mgmtport-no-snat-subnets-v4 and mgmtport-no-snat-subnets-v6
nftables set based on node's annotation values.

Signed-off-by: Yossi Boaron <[email protected]>

Author: yboaron

go-controller/pkg/node/managementport/port_linux.go

@@ -485,6 +485,57 @@ func setupManagementPortNFTChain(interfaceName string, cfg *managementPortConfig
 	return nil
 }
 
+func UpdateNoSNATSubnetsSets(node *corev1.Node, getSubnetsFn func(*corev1.Node) ([]string, error)) error {
+	subnetsList, err := getSubnetsFn(node)
+	if err != nil {
+		return fmt.Errorf("error retrieving subnets list: %w", err)
+	}
+
+	subNetV4 := make([]*knftables.Element, 0)
+	subNetV6 := make([]*knftables.Element, 0)
+
+	for _, subnet := range subnetsList {
+		if utilnet.IPFamilyOfCIDRString(subnet) == utilnet.IPv4 {
+			subNetV4 = append(subNetV4,
+				&knftables.Element{
+					Set: types.NFTMgmtPortNoSNATSubnetsV4,
+					Key: []string{subnet},
+				},
+			)
+		}
+		if utilnet.IPFamilyOfCIDRString(subnet) == utilnet.IPv6 {
+			subNetV6 = append(subNetV6,
+				&knftables.Element{
+					Set: types.NFTMgmtPortNoSNATSubnetsV6,
+					Key: []string{subnet},
+				},
+			)
+		}
+
+	}
+	nft, err := nodenft.GetNFTablesHelper()
+	if err != nil {
+		return fmt.Errorf("failed to get nftables: %v", err)
+	}
+
+	tx := nft.NewTransaction()
+	tx.Flush(&knftables.Set{
+		Name: types.NFTMgmtPortNoSNATSubnetsV4,
+	})
+	tx.Flush(&knftables.Set{
+		Name: types.NFTMgmtPortNoSNATSubnetsV6,
+	})
+
+	for _, elem := range subNetV4 {
+		tx.Add(elem)
+	}
+	for _, elem := range subNetV6 {
+		tx.Add(elem)
+	}
+
+	return nft.Run(context.TODO(), tx)
+}
+
 // createPlatformManagementPort creates a management port attached to the node switch
 // that lets the node access its pods via their private IP address. This is used
 // for health checking and other management tasks.

go-controller/pkg/node/obj_retry_node.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory"
+	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/managementport"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/retry"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util"
@@ -121,7 +122,7 @@ func (h *nodeEventHandler) AreResourcesEqual(obj1, obj2 interface{}) (bool, erro
 		if !ok {
 			return false, fmt.Errorf("could not cast obj2 of type %T to *kapi.Node", obj2)
 		}
-		return reflect.DeepEqual(node1.Status.Addresses, node2.Status.Addresses), nil
+		return reflect.DeepEqual(node1.Status.Addresses, node2.Status.Addresses) && reflect.DeepEqual(node1.Annotations, node2.Annotations), nil
 
 	default:
 		return false, fmt.Errorf("no object comparison for type %s", h.objType)
@@ -175,6 +176,13 @@ func (h *nodeEventHandler) AddResource(obj interface{}, _ bool) error {
 		node := obj.(*corev1.Node)
 		// if it's our node that is changing, then nothing to do as we dont add our own IP to the nftables rules
 		if node.Name == h.nc.name {
+			if util.NodeDontSNATSubnetAnnotationExist(node) {
+				err := managementport.UpdateNoSNATSubnetsSets(node, util.ParseNodeDontSNATSubnetsList)
+				if err != nil {
+					return fmt.Errorf("error updating no snat subnets sets: %w", err)
+				}
+			}
+
 			return nil
 		}
 		return h.nc.addOrUpdateNode(node)
@@ -218,6 +226,15 @@ func (h *nodeEventHandler) UpdateResource(oldObj, newObj interface{}, _ bool) er
 
 		// if it's our node that is changing, then nothing to do as we dont add our own IP to the nftables rules
 		if newNode.Name == h.nc.name {
+
+			// if node's dont SNAT subnet annotation changed sync nftables
+			if !reflect.DeepEqual(oldNode.Annotations, newNode.Annotations) &&
+				util.NodeDontSNATSubnetAnnotationChanged(oldNode, newNode) {
+				err := managementport.UpdateNoSNATSubnetsSets(newNode, util.ParseNodeDontSNATSubnetsList)
+				if err != nil {
+					return fmt.Errorf("error updating no snat subnets sets: %w", err)
+				}
+			}
 			return nil
 		}
 
@@ -273,6 +290,10 @@ func (h *nodeEventHandler) DeleteResource(obj, _ interface{}) error {
 
 	case factory.NodeType:
 		h.nc.deleteNode(obj.(*corev1.Node))
+		_ = managementport.UpdateNoSNATSubnetsSets(obj.(*corev1.Node), func(_ *corev1.Node) ([]string, error) {
+			return []string{}, nil
+		})
+
 		return nil
 
 	default:

go-controller/pkg/ovnwebhook/nodeadmission.go

@@ -34,6 +34,7 @@ var commonNodeAnnotationChecks = map[string]checkNodeAnnot{
 	util.OvnNodeMasqCIDR:                   nil,
 	util.OvnNodeGatewayMtuSupport:          nil,
 	util.OvnNodeManagementPort:             nil,
+	util.OvnNodeDontSNATSubnets:            nil,
 	util.OvnNodeChassisID: func(v annotationChange, _ string) error {
 		if v.action == removed {
 			return fmt.Errorf("%s cannot be removed", util.OvnNodeChassisID)

go-controller/pkg/util/node_annotations.go

@@ -154,6 +154,9 @@ const (
 
 	// ovnNodeEncapIPs is used to indicate encap IPs set on the node
 	OVNNodeEncapIPs = "k8s.ovn.org/node-encap-ips"
+
+	// OvnNodeDontSNATSubnets is a user assigned source subnets that should avoid SNAT at ovn-k8s-mp0 interface
+	OvnNodeDontSNATSubnets = "k8s.ovn.org/node-ingress-snat-exclude-subnets"
 )
 
 type L3GatewayConfig struct {
@@ -1115,15 +1118,45 @@ func ParseNodeHostCIDRsExcludeOVNNetworks(node *corev1.Node) ([]string, error) {
 }
 
 func ParseNodeHostCIDRsList(node *corev1.Node) ([]string, error) {
-	addrAnnotation, ok := node.Annotations[OVNNodeHostCIDRs]
+	return parseNodeAnnotationList(node, OVNNodeHostCIDRs)
+}
+
+func ParseNodeDontSNATSubnetsList(node *corev1.Node) ([]string, error) {
+	return parseNodeAnnotationList(node, OvnNodeDontSNATSubnets)
+}
+
+// NodeDontSNATSubnetAnnotationChanged returns true if the OvnNodeDontSNATSubnets in the corev1.Nodes doesn't match
+func NodeDontSNATSubnetAnnotationChanged(oldNode, newNode *corev1.Node) bool {
+	oldVal, oldOk := oldNode.Annotations[OvnNodeDontSNATSubnets]
+	newVal, newOk := newNode.Annotations[OvnNodeDontSNATSubnets]
+
+	if oldOk != newOk {
+		return true
+	}
+
+	if oldOk && newOk && oldVal != newVal {
+		return true
+	}
+
+	return false
+}
+
+// NodeDontSNATSubnetAnnotationExist returns true OvnNodeDontSNATSubnets annotation key exists in node annotation
+func NodeDontSNATSubnetAnnotationExist(node *corev1.Node) bool {
+	_, ok := node.Annotations[OvnNodeDontSNATSubnets]
+	return ok
+}
+
+func parseNodeAnnotationList(node *corev1.Node, annotationKey string) ([]string, error) {
+	annotationValue, ok := node.Annotations[annotationKey]
 	if !ok {
-		return nil, newAnnotationNotSetError("%s annotation not found for node %q", OVNNodeHostCIDRs, node.Name)
+		return []string{}, nil
 	}
 
 	var cfg []string
-	if err := json.Unmarshal([]byte(addrAnnotation), &cfg); err != nil {
-		return nil, fmt.Errorf("failed to unmarshal host cidrs annotation %s for node %q: %v",
-			addrAnnotation, node.Name, err)
+	if err := json.Unmarshal([]byte(annotationValue), &cfg); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal %s annotation %s for node %q: %v",
+			annotationKey, annotationValue, node.Name, err)
 	}
 	return cfg, nil
 }