Set host_routes after plugging the subnet into the router

the previous workflow was
- create a subnet with a host route via a fixed ip
- create a port with the fixed ip
- plug it into the router

but in IPv6, fixed ip are denied when automatic addresses are enabled

now the code does:
- create a subnet
- create a port
- plug the port into the router
- set the ip of the port as a gateway of the subnet

Author: gthiemonge

pkg/octavia/lb_mgmt_network.go

@@ -35,7 +35,6 @@ import (
 type NetworkProvisioningSummary struct {
 	TenantNetworkID         string
 	TenantSubnetID          string
-	TenantRouterPortID      string
 	ProviderNetworkID       string
 	RouterID                string
 	SecurityGroupID         string
@@ -48,7 +47,7 @@ type NetworkProvisioningSummary struct {
 // status.
 //
 
-func findPort(client *gophercloud.ServiceClient, networkID string, ipAddress string, log *logr.Logger) (*ports.Port, error) {
+func findPort(client *gophercloud.ServiceClient, networkID string, name string, log *logr.Logger) (*ports.Port, error) {
 	listOpts := ports.ListOpts{
 		NetworkID: networkID,
 	}
@@ -65,18 +64,16 @@ func findPort(client *gophercloud.ServiceClient, networkID string, ipAddress str
 	}
 	if len(allPorts) > 0 {
 		for _, port := range allPorts {
-			if len(port.FixedIPs) > 0 && port.FixedIPs[0].IPAddress == ipAddress {
+			if port.Name == name {
 				return &port, nil
 			}
 		}
 	}
 	return nil, nil
 }
 
-func ensurePort(client *gophercloud.ServiceClient, tenantNetwork *networks.Network, tenantSubnet *subnets.Subnet,
-	securityGroups *[]string, networkParameters *NetworkParameters, log *logr.Logger) (*ports.Port, error) {
-	ipAddress := networkParameters.TenantGateway.String()
-	p, err := findPort(client, tenantNetwork.ID, ipAddress, log)
+func ensurePort(client *gophercloud.ServiceClient, tenantNetwork *networks.Network, securityGroups *[]string, log *logr.Logger) (*ports.Port, error) {
+	p, err := findPort(client, tenantNetwork.ID, LbMgmtRouterPortName, log)
 	if err != nil {
 		return nil, err
 	}
@@ -89,15 +86,9 @@ func ensurePort(client *gophercloud.ServiceClient, tenantNetwork *networks.Netwo
 	log.Info("Unable to locate port, creating new one")
 	asu := true
 	createOpts := ports.CreateOpts{
-		Name:         LbMgmtRouterPortName,
-		AdminStateUp: &asu,
-		NetworkID:    tenantNetwork.ID,
-		FixedIPs: []ports.IP{
-			{
-				SubnetID:  tenantSubnet.ID,
-				IPAddress: ipAddress,
-			},
-		},
+		Name:           LbMgmtRouterPortName,
+		AdminStateUp:   &asu,
+		NetworkID:      tenantNetwork.ID,
 		SecurityGroups: securityGroups,
 	}
 	p, err = ports.Create(client, createOpts).Extract()
@@ -296,13 +287,19 @@ func ensureProvSubnet(
 	log *logr.Logger,
 ) (*subnets.Subnet, error) {
 	gatewayIP := ""
+	var ipVersion int
+	if networkParameters.ProviderCIDR.Addr().Is6() {
+		ipVersion = 6
+	} else {
+		ipVersion = 4
+	}
 	createOpts := subnets.CreateOpts{
 		Name:        LbProvSubnetName,
 		Description: LbProvSubnetDescription,
 		NetworkID:   providerNetwork.ID,
 		TenantID:    providerNetwork.TenantID,
 		CIDR:        networkParameters.ProviderCIDR.String(),
-		IPVersion:   gophercloud.IPVersion(4),
+		IPVersion:   gophercloud.IPVersion(ipVersion),
 		AllocationPools: []subnets.AllocationPool{
 			{
 				Start: networkParameters.ProviderAllocationStart.String(),
@@ -311,7 +308,7 @@ func ensureProvSubnet(
 		},
 		GatewayIP: &gatewayIP,
 	}
-	return ensureSubnet(client, 4, createOpts, log)
+	return ensureSubnet(client, ipVersion, createOpts, log)
 }
 
 func ensureProvNetwork(client *gophercloud.ServiceClient, netDetails *octaviav1.OctaviaLbMgmtNetworks, serviceTenantID string, log *logr.Logger) (
@@ -337,6 +334,31 @@ func ensureProvNetwork(client *gophercloud.ServiceClient, netDetails *octaviav1.
 	return provNet, nil
 }
 
+func ensureLbMgmtSubnetRoutes(
+	client *gophercloud.ServiceClient,
+	tenantSubnet *subnets.Subnet,
+	networkParameters *NetworkParameters,
+	tenantRouterPort *ports.Port,
+) error {
+	if len(tenantSubnet.HostRoutes) == 0 {
+		hostRoutes := []subnets.HostRoute{
+			{
+				DestinationCIDR: networkParameters.ProviderCIDR.String(),
+				NextHop:         tenantRouterPort.FixedIPs[0].IPAddress,
+			},
+		}
+		updateOpts := subnets.UpdateOpts{
+			HostRoutes: &hostRoutes,
+		}
+		_, err := subnets.Update(client, tenantSubnet.ID, updateOpts).Extract()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func ensureLbMgmtSubnet(
 	client *gophercloud.ServiceClient,
 	tenantNetwork *networks.Network,
@@ -369,7 +391,6 @@ func ensureLbMgmtSubnet(
 				},
 			},
 			GatewayIP: &gatewayIP,
-			// TODO(beagles): ipv6 host routes
 		}
 	} else {
 		gatewayIP := LbMgmtSubnetGatewayIP
@@ -386,12 +407,6 @@ func ensureLbMgmtSubnet(
 					End:   networkParameters.TenantAllocationEnd.String(),
 				},
 			},
-			HostRoutes: []subnets.HostRoute{
-				{
-					DestinationCIDR: networkParameters.ProviderCIDR.String(),
-					NextHop:         networkParameters.TenantGateway.String(),
-				},
-			},
 			GatewayIP: &gatewayIP,
 		}
 	}
@@ -797,7 +812,7 @@ func EnsureAmphoraManagementNetwork(
 
 	securityGroups := []string{lbMgmtSecurityGroupID, lbHealthSecurityGroupID}
 
-	tenantRouterPort, err := ensurePort(client, tenantNetwork, tenantSubnet, &securityGroups, networkParameters, log)
+	tenantRouterPort, err := ensurePort(client, tenantNetwork, &securityGroups, log)
 	if err != nil {
 		return NetworkProvisioningSummary{}, err
 	}
@@ -856,25 +871,25 @@ func EnsureAmphoraManagementNetwork(
 			log.Error(err, "Unable to create router object")
 			return NetworkProvisioningSummary{}, err
 		}
-	}
-	if tenantRouterPort.DeviceID == "" {
+
 		interfaceOpts := routers.AddInterfaceOpts{
 			PortID: tenantRouterPort.ID,
 		}
 		_, err := routers.AddInterface(client, router.ID, interfaceOpts).Extract()
 		if err != nil {
 			log.Error(err, fmt.Sprintf("Unable to add interface port %s to router %s", tenantRouterPort.ID, router.ID))
 		}
-	} else if tenantRouterPort.DeviceID != router.ID {
-		return NetworkProvisioningSummary{},
-			fmt.Errorf("Port %s has unexpected device ID %s and cannot be added to router %s", tenantRouterPort.ID,
-				tenantRouterPort.DeviceID, router.ID)
+	}
+	// Set route on subnet
+
+	err = ensureLbMgmtSubnetRoutes(client, tenantSubnet, networkParameters, tenantRouterPort)
+	if err != nil {
+		log.Error(err, fmt.Sprintf("Unable to set host routes on subnet %s", tenantSubnet.ID))
 	}
 
 	return NetworkProvisioningSummary{
 		TenantNetworkID:         tenantNetwork.ID,
 		TenantSubnetID:          tenantSubnet.ID,
-		TenantRouterPortID:      tenantRouterPort.ID,
 		ProviderNetworkID:       providerNetwork.ID,
 		RouterID:                router.ID,
 		SecurityGroupID:         lbMgmtSecurityGroupID,

pkg/octavia/network_parameters.go

@@ -17,7 +17,6 @@ type NetworkParameters struct {
 	TenantCIDR              netip.Prefix
 	TenantAllocationStart   netip.Addr
 	TenantAllocationEnd     netip.Addr
-	TenantGateway           netip.Addr
 }
 
 // NADConfig - IPAM parameters of the NAD
@@ -51,17 +50,15 @@ func getConfigFromNAD(
 	return nadConfig, nil
 }
 
-func getRangeAndGatewayFromCIDR(
+func getRangeFromCIDR(
 	cidr netip.Prefix,
-) (start netip.Addr, end netip.Addr, gateway netip.Addr) {
+) (start netip.Addr, end netip.Addr) {
 	addr := cidr.Addr()
 	if addr.Is6() {
 		addrBytes := addr.As16()
 		for i := 8; i < 15; i++ {
 			addrBytes[i] = 0
 		}
-		addrBytes[15] = 3
-		gateway = netip.AddrFrom16(addrBytes)
 		addrBytes[15] = 5
 		start = netip.AddrFrom16(addrBytes)
 		for i := 8; i < 15; i++ {
@@ -72,8 +69,6 @@ func getRangeAndGatewayFromCIDR(
 	} else {
 		addrBytes := addr.As4()
 		addrBytes[2] = 0
-		addrBytes[3] = 3
-		gateway = netip.AddrFrom4(addrBytes)
 		addrBytes[3] = 5
 		start = netip.AddrFrom4(addrBytes)
 		addrBytes[2] = 0xff
@@ -124,10 +119,9 @@ func GetNetworkParametersFromNAD(
 		return nil, fmt.Errorf("the tenant CIDR is /%d, it should be /%d", networkParameters.TenantCIDR.Bits(), bitlen)
 	}
 
-	start, end, gateway := getRangeAndGatewayFromCIDR(networkParameters.TenantCIDR)
+	start, end := getRangeFromCIDR(networkParameters.TenantCIDR)
 	networkParameters.TenantAllocationStart = start
 	networkParameters.TenantAllocationEnd = end
-	networkParameters.TenantGateway = gateway
 
 	return networkParameters, err
 }