Configures ephemeral port range for OVN SNAT'ing

There was a previous bug where when an egress packet would be SNAT'ed to
the node IP, using a nodeport source port, it would cause reply traffic
to get DNAT'ed to the nodeport load balancer. This happened because the
egress connections were not conntracked correctly.

This was fixed via:

https://issues.redhat.com/browse/OCPBUGS-25889
https://issues.redhat.com/browse/FDP-291

However, that fix was not hardware offloadable. The ideal fix here would
be to always commit to conntrack and have it be HW offloadable. Until we
have a better solution, we can configure the port range for OVN to use
on its SNAT. This applies to all SNATs for traffic that enters the local
host or leaves the host.

The new config option --ephemeral-port-range "<minPort>-<maxPort>" can
be used to specify the port range to use with OVN. If not provided, this
value will be automatically derived from the ephemeral port range in
/proc/sys/net/ipv4/ip_local_port_range, which is typically set already
to avoid nodeport range conflicts.

Signed-off-by: Tim Rozet <[email protected]>

Author: trozet

go-controller/pkg/config/config.go

@@ -38,6 +38,9 @@ const DefaultVXLANPort = 4789
 
 const DefaultDBTxnTimeout = time.Second * 100
 
+// DefaultEphemeralPortRange is used for unit testing only
+const DefaultEphemeralPortRange = "32768-60999"
+
 // The following are global config parameters that other modules may access directly
 var (
 	// Build information. Populated at build-time.
@@ -494,6 +497,10 @@ type GatewayConfig struct {
 	DisableForwarding bool `gcfg:"disable-forwarding"`
 	// AllowNoUplink (disabled by default) controls if the external gateway bridge without an uplink port is allowed in local gateway mode.
 	AllowNoUplink bool `gcfg:"allow-no-uplink"`
+	// EphemeralPortRange is the range of ports used by egress SNAT operations in OVN. Specifically for NAT where
+	// the source IP of the NAT will be a shared Node IP address. If unset, the value will be determined by sysctl lookup
+	// for the kernel's ephemeral range: net.ipv4.ip_local_port_range. Format is "<min port>-<max port>".
+	EphemeralPortRange string `gfcg:"ephemeral-port-range"`
 }
 
 // OvnAuthConfig holds client authentication and location details for
@@ -664,6 +671,9 @@ func PrepareTestConfig() error {
 	Kubernetes.DisableRequestedChassis = false
 	EnableMulticast = false
 	Default.OVSDBTxnTimeout = 5 * time.Second
+	if Gateway.Mode != GatewayModeDisabled {
+		Gateway.EphemeralPortRange = DefaultEphemeralPortRange
+	}
 
 	if err := completeConfig(); err != nil {
 		return err
@@ -1509,6 +1519,14 @@ var OVNGatewayFlags = []cli.Flag{
 		Usage:       "Allow the external gateway bridge without an uplink port in local gateway mode",
 		Destination: &cliConfig.Gateway.AllowNoUplink,
 	},
+	&cli.StringFlag{
+		Name: "ephemeral-port-range",
+		Usage: "The port range in '<min port>-<max port>' format for OVN to use when SNAT'ing to a node IP. " +
+			"This range should not collide with the node port range being used in Kubernetes. If not provided, " +
+			"the default value will be derived from checking the sysctl value of net.ipv4.ip_local_port_range on the node.",
+		Destination: &cliConfig.Gateway.EphemeralPortRange,
+		Value:       Gateway.EphemeralPortRange,
+	},
 	// Deprecated CLI options
 	&cli.BoolFlag{
 		Name:        "init-gateways",
@@ -1917,6 +1935,19 @@ func buildGatewayConfig(ctx *cli.Context, cli, file *config) error {
 		if !found {
 			return fmt.Errorf("invalid gateway mode %q: expect one of %s", string(Gateway.Mode), strings.Join(validModes, ","))
 		}
+
+		if len(Gateway.EphemeralPortRange) > 0 {
+			if !isValidEphemeralPortRange(Gateway.EphemeralPortRange) {
+				return fmt.Errorf("invalid ephemeral-port-range, should be in the format <min port>-<max port>")
+			}
+		} else {
+			// auto-detect ephermal range
+			portRange, err := getKernelEphemeralPortRange()
+			if err != nil {
+				return fmt.Errorf("unable to auto-detect ephemeral port range to use with OVN")
+			}
+			Gateway.EphemeralPortRange = portRange
+		}
 	}
 
 	// Options are only valid if Mode is not disabled
@@ -1927,6 +1958,9 @@ func buildGatewayConfig(ctx *cli.Context, cli, file *config) error {
 		if Gateway.NextHop != "" {
 			return fmt.Errorf("gateway next-hop option %q not allowed when gateway is disabled", Gateway.NextHop)
 		}
+		if len(Gateway.EphemeralPortRange) > 0 {
+			return fmt.Errorf("gateway ephemeral port range option not allowed when gateway is disabled")
+		}
 	}
 
 	if Gateway.Mode != GatewayModeShared && Gateway.VLANID != 0 {

go-controller/pkg/config/utils.go

@@ -3,7 +3,9 @@ package config
 import (
 	"fmt"
 	"net"
+	"os"
 	"reflect"
+	"regexp"
 	"strconv"
 	"strings"
 
@@ -328,3 +330,49 @@ func AllocateV6MasqueradeIPs(masqueradeSubnetNetworkAddress net.IP, masqueradeIP
 	}
 	return nil
 }
+
+func isValidEphemeralPortRange(s string) bool {
+	// Regex to match "<number>-<number>" with no extra characters
+	re := regexp.MustCompile(`^(\d{1,5})-(\d{1,5})$`)
+	matches := re.FindStringSubmatch(s)
+	if matches == nil {
+		return false
+	}
+
+	minPort, err1 := strconv.Atoi(matches[1])
+	maxPort, err2 := strconv.Atoi(matches[2])
+	if err1 != nil || err2 != nil {
+		return false
+	}
+
+	// Port numbers must be in the 1-65535 range
+	if minPort < 1 || minPort > 65535 || maxPort < 0 || maxPort > 65535 {
+		return false
+	}
+
+	return maxPort > minPort
+}
+
+func getKernelEphemeralPortRange() (string, error) {
+	data, err := os.ReadFile("/proc/sys/net/ipv4/ip_local_port_range")
+	if err != nil {
+		return "", fmt.Errorf("failed to read port range: %w", err)
+	}
+
+	parts := strings.Fields(string(data))
+	if len(parts) != 2 {
+		return "", fmt.Errorf("unexpected format: %q", string(data))
+	}
+
+	minPort, err := strconv.Atoi(parts[0])
+	if err != nil {
+		return "", fmt.Errorf("invalid min port: %w", err)
+	}
+
+	maxPort, err := strconv.Atoi(parts[1])
+	if err != nil {
+		return "", fmt.Errorf("invalid max port: %w", err)
+	}
+
+	return fmt.Sprintf("%d-%d", minPort, maxPort), nil
+}

go-controller/pkg/libovsdb/ops/router.go

@@ -961,6 +961,10 @@ func buildNAT(
 		Match:       match,
 	}
 
+	if config.Gateway.Mode != config.GatewayModeDisabled {
+		nat.ExternalPortRange = config.Gateway.EphemeralPortRange
+	}
+
 	if logicalPort != "" {
 		nat.LogicalPort = &logicalPort
 	}
@@ -1061,7 +1065,7 @@ func isEquivalentNAT(existing *nbdb.NAT, searched *nbdb.NAT) bool {
 		return false
 	}
 
-	// Compre externalIP if its not empty.
+	// Compare externalIP if it's not empty.
 	if searched.ExternalIP != "" && searched.ExternalIP != existing.ExternalIP {
 		return false
 	}

go-controller/pkg/ovn/gateway_test.go

@@ -220,27 +220,35 @@ func generateGatewayInitExpectedNB(testData []libovsdbtest.TestData, expectedOVN
 			natUUID := fmt.Sprintf("nat-%d-UUID", i)
 			natUUIDs = append(natUUIDs, natUUID)
 			physicalIP, _ := util.MatchFirstIPNetFamily(utilnet.IsIPv6CIDR(subnet), l3GatewayConfig.IPAddresses)
-			testData = append(testData, &nbdb.NAT{
+			nat := nbdb.NAT{
 				UUID:       natUUID,
 				ExternalIP: physicalIP.IP.String(),
 				LogicalIP:  subnet.String(),
 				Options:    map[string]string{"stateless": "false"},
 				Type:       nbdb.NATTypeSNAT,
-			})
+			}
+			if config.Gateway.Mode != config.GatewayModeDisabled {
+				nat.ExternalPortRange = config.DefaultEphemeralPortRange
+			}
+			testData = append(testData, &nat)
 		}
 	}
 
 	for i, physicalIP := range l3GatewayConfig.IPAddresses {
 		natUUID := fmt.Sprintf("nat-join-%d-UUID", i)
 		natUUIDs = append(natUUIDs, natUUID)
 		joinLRPIP, _ := util.MatchFirstIPNetFamily(utilnet.IsIPv6CIDR(physicalIP), joinLRPIPs)
-		testData = append(testData, &nbdb.NAT{
+		nat := nbdb.NAT{
 			UUID:       natUUID,
 			ExternalIP: physicalIP.IP.String(),
 			LogicalIP:  joinLRPIP.IP.String(),
 			Options:    map[string]string{"stateless": "false"},
 			Type:       nbdb.NATTypeSNAT,
-		})
+		}
+		if config.Gateway.Mode != config.GatewayModeDisabled {
+			nat.ExternalPortRange = config.DefaultEphemeralPortRange
+		}
+		testData = append(testData, &nat)
 	}
 
 	testData = append(testData, &nbdb.MeterBand{
@@ -394,6 +402,7 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() {
 	ginkgo.Context("Gateway Creation Operations Shared Gateway Mode", func() {
 		ginkgo.BeforeEach(func() {
 			config.Gateway.Mode = config.GatewayModeShared
+			config.Gateway.EphemeralPortRange = config.DefaultEphemeralPortRange
 		})
 
 		ginkgo.It("creates an IPv4 gateway in OVN", func() {
@@ -1441,6 +1450,7 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() {
 		ginkgo.BeforeEach(func() {
 			config.Gateway.Mode = config.GatewayModeLocal
 			config.IPv6Mode = false
+			config.Gateway.EphemeralPortRange = config.DefaultEphemeralPortRange
 		})
 
 		ginkgo.It("creates a dual-stack gateway in OVN", func() {

go-controller/pkg/ovn/namespace_test.go

@@ -238,6 +238,7 @@ var _ = ginkgo.Describe("OVN Namespace Operations", func() {
 		ginkgo.It("creates an address set for existing nodes when the host network traffic namespace is created", func() {
 			config.Gateway.Mode = config.GatewayModeShared
 			config.Gateway.NodeportEnable = true
+			config.Gateway.EphemeralPortRange = config.DefaultEphemeralPortRange
 			var err error
 			config.Default.ClusterSubnets, err = config.ParseClusterSubnetEntries(clusterCIDR)
 			gomega.Expect(err).NotTo(gomega.HaveOccurred())