Merge pull request #5265 from trozet/specify_ovn_ephemeral_port_range

Configures ephemeral port range for OVN SNAT'ing

Author: trozet

go-controller/pkg/config/config.go

@@ -38,6 +38,9 @@ const DefaultVXLANPort = 4789
 
 const DefaultDBTxnTimeout = time.Second * 100
 
+// DefaultEphemeralPortRange is used for unit testing only
+const DefaultEphemeralPortRange = "32768-60999"
+
 // The following are global config parameters that other modules may access directly
 var (
 	// Build information. Populated at build-time.
@@ -494,6 +497,10 @@ type GatewayConfig struct {
 	DisableForwarding bool `gcfg:"disable-forwarding"`
 	// AllowNoUplink (disabled by default) controls if the external gateway bridge without an uplink port is allowed in local gateway mode.
 	AllowNoUplink bool `gcfg:"allow-no-uplink"`
+	// EphemeralPortRange is the range of ports used by egress SNAT operations in OVN. Specifically for NAT where
+	// the source IP of the NAT will be a shared Node IP address. If unset, the value will be determined by sysctl lookup
+	// for the kernel's ephemeral range: net.ipv4.ip_local_port_range. Format is "<min port>-<max port>".
+	EphemeralPortRange string `gfcg:"ephemeral-port-range"`
 }
 
 // OvnAuthConfig holds client authentication and location details for
@@ -664,6 +671,9 @@ func PrepareTestConfig() error {
 	Kubernetes.DisableRequestedChassis = false
 	EnableMulticast = false
 	Default.OVSDBTxnTimeout = 5 * time.Second
+	if Gateway.Mode != GatewayModeDisabled {
+		Gateway.EphemeralPortRange = DefaultEphemeralPortRange
+	}
 
 	if err := completeConfig(); err != nil {
 		return err
@@ -1509,6 +1519,14 @@ var OVNGatewayFlags = []cli.Flag{
 		Usage:       "Allow the external gateway bridge without an uplink port in local gateway mode",
 		Destination: &cliConfig.Gateway.AllowNoUplink,
 	},
+	&cli.StringFlag{
+		Name: "ephemeral-port-range",
+		Usage: "The port range in '<min port>-<max port>' format for OVN to use when SNAT'ing to a node IP. " +
+			"This range should not collide with the node port range being used in Kubernetes. If not provided, " +
+			"the default value will be derived from checking the sysctl value of net.ipv4.ip_local_port_range on the node.",
+		Destination: &cliConfig.Gateway.EphemeralPortRange,
+		Value:       Gateway.EphemeralPortRange,
+	},
 	// Deprecated CLI options
 	&cli.BoolFlag{
 		Name:        "init-gateways",
@@ -1917,6 +1935,19 @@ func buildGatewayConfig(ctx *cli.Context, cli, file *config) error {
 		if !found {
 			return fmt.Errorf("invalid gateway mode %q: expect one of %s", string(Gateway.Mode), strings.Join(validModes, ","))
 		}
+
+		if len(Gateway.EphemeralPortRange) > 0 {
+			if !isValidEphemeralPortRange(Gateway.EphemeralPortRange) {
+				return fmt.Errorf("invalid ephemeral-port-range, should be in the format <min port>-<max port>")
+			}
+		} else {
+			// auto-detect ephermal range
+			portRange, err := getKernelEphemeralPortRange()
+			if err != nil {
+				return fmt.Errorf("unable to auto-detect ephemeral port range to use with OVN")
+			}
+			Gateway.EphemeralPortRange = portRange
+		}
 	}
 
 	// Options are only valid if Mode is not disabled
@@ -1927,6 +1958,9 @@ func buildGatewayConfig(ctx *cli.Context, cli, file *config) error {
 		if Gateway.NextHop != "" {
 			return fmt.Errorf("gateway next-hop option %q not allowed when gateway is disabled", Gateway.NextHop)
 		}
+		if len(Gateway.EphemeralPortRange) > 0 {
+			return fmt.Errorf("gateway ephemeral port range option not allowed when gateway is disabled")
+		}
 	}
 
 	if Gateway.Mode != GatewayModeShared && Gateway.VLANID != 0 {

go-controller/pkg/config/utils.go

@@ -3,7 +3,9 @@ package config
 import (
 	"fmt"
 	"net"
+	"os"
 	"reflect"
+	"regexp"
 	"strconv"
 	"strings"
 
@@ -328,3 +330,49 @@ func AllocateV6MasqueradeIPs(masqueradeSubnetNetworkAddress net.IP, masqueradeIP
 	}
 	return nil
 }
+
+func isValidEphemeralPortRange(s string) bool {
+	// Regex to match "<number>-<number>" with no extra characters
+	re := regexp.MustCompile(`^(\d{1,5})-(\d{1,5})$`)
+	matches := re.FindStringSubmatch(s)
+	if matches == nil {
+		return false
+	}
+
+	minPort, err1 := strconv.Atoi(matches[1])
+	maxPort, err2 := strconv.Atoi(matches[2])
+	if err1 != nil || err2 != nil {
+		return false
+	}
+
+	// Port numbers must be in the 1-65535 range
+	if minPort < 1 || minPort > 65535 || maxPort < 0 || maxPort > 65535 {
+		return false
+	}
+
+	return maxPort > minPort
+}
+
+func getKernelEphemeralPortRange() (string, error) {
+	data, err := os.ReadFile("/proc/sys/net/ipv4/ip_local_port_range")
+	if err != nil {
+		return "", fmt.Errorf("failed to read port range: %w", err)
+	}
+
+	parts := strings.Fields(string(data))
+	if len(parts) != 2 {
+		return "", fmt.Errorf("unexpected format: %q", string(data))
+	}
+
+	minPort, err := strconv.Atoi(parts[0])
+	if err != nil {
+		return "", fmt.Errorf("invalid min port: %w", err)
+	}
+
+	maxPort, err := strconv.Atoi(parts[1])
+	if err != nil {
+		return "", fmt.Errorf("invalid max port: %w", err)
+	}
+
+	return fmt.Sprintf("%d-%d", minPort, maxPort), nil
+}

go-controller/pkg/libovsdb/ops/router.go

@@ -961,6 +961,10 @@ func buildNAT(
 		Match:       match,
 	}
 
+	if config.Gateway.Mode != config.GatewayModeDisabled {
+		nat.ExternalPortRange = config.Gateway.EphemeralPortRange
+	}
+
 	if logicalPort != "" {
 		nat.LogicalPort = &logicalPort
 	}
@@ -1061,7 +1065,7 @@ func isEquivalentNAT(existing *nbdb.NAT, searched *nbdb.NAT) bool {
 		return false
 	}
 
-	// Compre externalIP if its not empty.
+	// Compare externalIP if it's not empty.
 	if searched.ExternalIP != "" && searched.ExternalIP != existing.ExternalIP {
 		return false
 	}

go-controller/pkg/ovn/gateway_test.go

@@ -220,27 +220,35 @@ func generateGatewayInitExpectedNB(testData []libovsdbtest.TestData, expectedOVN
 			natUUID := fmt.Sprintf("nat-%d-UUID", i)
 			natUUIDs = append(natUUIDs, natUUID)
 			physicalIP, _ := util.MatchFirstIPNetFamily(utilnet.IsIPv6CIDR(subnet), l3GatewayConfig.IPAddresses)
-			testData = append(testData, &nbdb.NAT{
+			nat := nbdb.NAT{
 				UUID:       natUUID,
 				ExternalIP: physicalIP.IP.String(),
 				LogicalIP:  subnet.String(),
 				Options:    map[string]string{"stateless": "false"},
 				Type:       nbdb.NATTypeSNAT,
-			})
+			}
+			if config.Gateway.Mode != config.GatewayModeDisabled {
+				nat.ExternalPortRange = config.DefaultEphemeralPortRange
+			}
+			testData = append(testData, &nat)
 		}
 	}
 
 	for i, physicalIP := range l3GatewayConfig.IPAddresses {
 		natUUID := fmt.Sprintf("nat-join-%d-UUID", i)
 		natUUIDs = append(natUUIDs, natUUID)
 		joinLRPIP, _ := util.MatchFirstIPNetFamily(utilnet.IsIPv6CIDR(physicalIP), joinLRPIPs)
-		testData = append(testData, &nbdb.NAT{
+		nat := nbdb.NAT{
 			UUID:       natUUID,
 			ExternalIP: physicalIP.IP.String(),
 			LogicalIP:  joinLRPIP.IP.String(),
 			Options:    map[string]string{"stateless": "false"},
 			Type:       nbdb.NATTypeSNAT,
-		})
+		}
+		if config.Gateway.Mode != config.GatewayModeDisabled {
+			nat.ExternalPortRange = config.DefaultEphemeralPortRange
+		}
+		testData = append(testData, &nat)
 	}
 
 	testData = append(testData, &nbdb.MeterBand{
@@ -394,6 +402,7 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() {
 	ginkgo.Context("Gateway Creation Operations Shared Gateway Mode", func() {
 		ginkgo.BeforeEach(func() {
 			config.Gateway.Mode = config.GatewayModeShared
+			config.Gateway.EphemeralPortRange = config.DefaultEphemeralPortRange
 		})
 
 		ginkgo.It("creates an IPv4 gateway in OVN", func() {
@@ -1441,6 +1450,7 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() {
 		ginkgo.BeforeEach(func() {
 			config.Gateway.Mode = config.GatewayModeLocal
 			config.IPv6Mode = false
+			config.Gateway.EphemeralPortRange = config.DefaultEphemeralPortRange
 		})
 
 		ginkgo.It("creates a dual-stack gateway in OVN", func() {

go-controller/pkg/ovn/namespace_test.go

@@ -238,6 +238,7 @@ var _ = ginkgo.Describe("OVN Namespace Operations", func() {
 		ginkgo.It("creates an address set for existing nodes when the host network traffic namespace is created", func() {
 			config.Gateway.Mode = config.GatewayModeShared
 			config.Gateway.NodeportEnable = true
+			config.Gateway.EphemeralPortRange = config.DefaultEphemeralPortRange
 			var err error
 			config.Default.ClusterSubnets, err = config.ParseClusterSubnetEntries(clusterCIDR)
 			gomega.Expect(err).NotTo(gomega.HaveOccurred())