Get IP addresses of neutron subports (kubernetes#2306)

Vlan-aware VMs are commonly used in telecom.
Those VMs are plugged into flat networks and use neutron trunk.
https://docs.openstack.org/neutron/latest/admin/config-trunking.html

Currenlty, nodeAddresses() only returns neutron ports directly attached to VM.
For vlan-aware VMs, IP addresses are assigned on neutron subports.
Subports are attached to the trunk; they are not attached to the VM directly.
This pull request changes nodeAddresses() to return IP addresses of neutron subports when they exist.
Without this change, Kubernetes is unable to IP addresses of vlan-aware VMs.
This change is transparent to VMs not using neturon trunk.

Signed-off-by: Jing Zhang <[email protected]>

Author: jingczhang

pkg/openstack/instances.go

@@ -29,7 +29,8 @@ import (
 	"github.com/gophercloud/gophercloud"
 	"github.com/gophercloud/gophercloud/openstack/compute/v2/flavors"
 	"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
-	"github.com/gophercloud/gophercloud/openstack/networking/v2/ports"
+	"github.com/gophercloud/gophercloud/openstack/networking/v2/networks"
+	neutronports "github.com/gophercloud/gophercloud/openstack/networking/v2/ports"
 	"github.com/gophercloud/gophercloud/pagination"
 	"github.com/mitchellh/mapstructure"
 	v1 "k8s.io/api/core/v1"
@@ -43,7 +44,6 @@ import (
 	"k8s.io/cloud-provider-openstack/pkg/util"
 	"k8s.io/cloud-provider-openstack/pkg/util/errors"
 	"k8s.io/cloud-provider-openstack/pkg/util/metadata"
-	"k8s.io/cloud-provider-openstack/pkg/util/openstack"
 )
 
 // Instances encapsulates an implementation of Instances for OpenStack.
@@ -240,7 +240,7 @@ func (i *Instances) NodeAddressesByProviderID(ctx context.Context, providerID st
 		return []v1.NodeAddress{}, err
 	}
 
-	addresses, err := nodeAddresses(server, ports, i.networkingOpts)
+	addresses, err := nodeAddresses(server, ports, i.network, i.networkingOpts)
 	if err != nil {
 		return []v1.NodeAddress{}, err
 	}
@@ -345,7 +345,7 @@ func (i *Instances) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloud
 	if err != nil {
 		return nil, err
 	}
-	addresses, err := nodeAddresses(srv, ports, i.networkingOpts)
+	addresses, err := nodeAddresses(srv, ports, i.network, i.networkingOpts)
 	if err != nil {
 		return nil, err
 	}
@@ -573,7 +573,7 @@ func getServerByName(client *gophercloud.ServiceClient, name types.NodeName) (*S
 // * access IPs
 // * metadata hostname
 // * server object Addresses (floating type)
-func nodeAddresses(srv *servers.Server, ports []ports.Port, networkingOpts NetworkingOpts) ([]v1.NodeAddress, error) {
+func nodeAddresses(srv *servers.Server, ports []PortWithTrunkDetails, client *gophercloud.ServiceClient, networkingOpts NetworkingOpts) ([]v1.NodeAddress, error) {
 	addrs := []v1.NodeAddress{}
 
 	// parse private IP addresses first in an ordered manner
@@ -633,6 +633,39 @@ func nodeAddresses(srv *servers.Server, ports []ports.Port, networkingOpts Netwo
 		return nil, err
 	}
 
+	// Add the addresses assigned on subports via trunk
+	// This exposes the vlan networks to which subports are attached
+	for _, port := range ports {
+		for _, subport := range port.TrunkDetails.SubPorts {
+			p, err := neutronports.Get(client, subport.PortID).Extract()
+			if err != nil {
+				klog.Errorf("Failed to get subport %s details: %v", subport.PortID, err)
+				continue
+			}
+			n, err := networks.Get(client, p.NetworkID).Extract()
+			if err != nil {
+				klog.Errorf("Failed to get subport %s network details: %v", subport.PortID, err)
+				continue
+			}
+			for _, fixedIP := range p.FixedIPs {
+				klog.V(5).Infof("Node '%s' is found subport '%s' address '%s/%s'", srv.Name, p.Name, n.Name, fixedIP.IPAddress)
+				isIPv6 := net.ParseIP(fixedIP.IPAddress).To4() == nil
+				if !(isIPv6 && networkingOpts.IPv6SupportDisabled) {
+					addr := Address{IPType: "fixed", Addr: fixedIP.IPAddress}
+					subportAddresses := map[string][]Address{n.Name: {addr}}
+					srvAddresses, ok := addresses[n.Name]
+					if !ok {
+						addresses[n.Name] = subportAddresses[n.Name]
+					} else {
+						// this is to take care the corner case
+						// where the same network is attached to the node both directly and via trunk
+						addresses[n.Name] = append(srvAddresses, subportAddresses[n.Name]...)
+					}
+				}
+			}
+		}
+	}
+
 	networks := make([]string, 0, len(addresses))
 	for k := range addresses {
 		networks = append(networks, k)
@@ -683,6 +716,7 @@ func nodeAddresses(srv *servers.Server, ports []ports.Port, networkingOpts Netwo
 		sortNodeAddresses(addrs, networkingOpts.AddressSortOrder)
 	}
 
+	klog.V(5).Infof("Node '%s' returns addresses '%v'", srv.Name, addrs)
 	return addrs, nil
 }
 
@@ -697,14 +731,25 @@ func getAddressesByName(client *gophercloud.ServiceClient, name types.NodeName,
 		return nil, err
 	}
 
-	return nodeAddresses(&srv.Server, ports, networkingOpts)
+	return nodeAddresses(&srv.Server, ports, client, networkingOpts)
 }
 
 // getAttachedPorts returns a list of ports attached to a server.
-func getAttachedPorts(client *gophercloud.ServiceClient, serverID string) ([]ports.Port, error) {
-	listOpts := ports.ListOpts{
+func getAttachedPorts(client *gophercloud.ServiceClient, serverID string) ([]PortWithTrunkDetails, error) {
+	listOpts := neutronports.ListOpts{
 		DeviceID: serverID,
 	}
 
-	return openstack.GetPorts(client, listOpts)
+	var ports []PortWithTrunkDetails
+
+	allPages, err := neutronports.List(client, listOpts).AllPages()
+	if err != nil {
+		return ports, err
+	}
+	err = neutronports.ExtractPortsInto(allPages, &ports)
+	if err != nil {
+		return ports, err
+	}
+
+	return ports, nil
 }

pkg/openstack/instancesv2.go

@@ -128,7 +128,7 @@ func (i *InstancesV2) InstanceMetadata(ctx context.Context, node *v1.Node) (*clo
 		return nil, err
 	}
 
-	addresses, err := nodeAddresses(&server.Server, ports, i.networkingOpts)
+	addresses, err := nodeAddresses(&server.Server, ports, i.network, i.networkingOpts)
 	if err != nil {
 		return nil, err
 	}

pkg/openstack/openstack.go

@@ -27,6 +27,8 @@ import (
 	"github.com/gophercloud/gophercloud"
 	"github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/availabilityzones"
 	"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
+	"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/trunk_details"
+	neutronports "github.com/gophercloud/gophercloud/openstack/networking/v2/ports"
 	"github.com/spf13/pflag"
 	gcfg "gopkg.in/gcfg.v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -67,6 +69,11 @@ func AddExtraFlags(fs *pflag.FlagSet) {
 	fs.StringArrayVar(&userAgentData, "user-agent", nil, "Extra data to add to gophercloud user-agent. Use multiple times to add more than one component.")
 }
 
+type PortWithTrunkDetails struct {
+	neutronports.Port
+	trunk_details.TrunkDetailsExt
+}
+
 // LoadBalancer is used for creating and maintaining load balancers
 type LoadBalancer struct {
 	secret  *gophercloud.ServiceClient

pkg/openstack/openstack_test.go

@@ -28,7 +28,7 @@ import (
 
 	"github.com/gophercloud/gophercloud"
 	"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
-	"github.com/gophercloud/gophercloud/openstack/networking/v2/ports"
+	neutronports "github.com/gophercloud/gophercloud/openstack/networking/v2/ports"
 	"github.com/spf13/pflag"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -373,10 +373,10 @@ func TestNodeAddresses(t *testing.T) {
 		PublicNetworkName: []string{"public"},
 	}
 
-	ports := []ports.Port{
-		{
+	ports := []PortWithTrunkDetails{{
+		Port: neutronports.Port{
 			Status: "ACTIVE",
-			FixedIPs: []ports.IP{
+			FixedIPs: []neutronports.IP{
 				{
 					IPAddress: "10.0.0.32",
 				},
@@ -385,9 +385,10 @@ func TestNodeAddresses(t *testing.T) {
 				},
 			},
 		},
+	},
 	}
 
-	addrs, err := nodeAddresses(&srv, ports, networkingOpts)
+	addrs, err := nodeAddresses(&srv, ports, nil, networkingOpts)
 	if err != nil {
 		t.Fatalf("nodeAddresses returned error: %v", err)
 	}
@@ -452,10 +453,10 @@ func TestNodeAddressesCustomPublicNetwork(t *testing.T) {
 		PublicNetworkName: []string{"pub-net"},
 	}
 
-	ports := []ports.Port{
-		{
+	ports := []PortWithTrunkDetails{{
+		Port: neutronports.Port{
 			Status: "ACTIVE",
-			FixedIPs: []ports.IP{
+			FixedIPs: []neutronports.IP{
 				{
 					IPAddress: "10.0.0.32",
 				},
@@ -464,9 +465,10 @@ func TestNodeAddressesCustomPublicNetwork(t *testing.T) {
 				},
 			},
 		},
+	},
 	}
 
-	addrs, err := nodeAddresses(&srv, ports, networkingOpts)
+	addrs, err := nodeAddresses(&srv, ports, nil, networkingOpts)
 	if err != nil {
 		t.Fatalf("nodeAddresses returned error: %v", err)
 	}
@@ -525,10 +527,10 @@ func TestNodeAddressesCustomPublicNetworkWithIntersectingFixedIP(t *testing.T) {
 		PublicNetworkName: []string{"pub-net"},
 	}
 
-	ports := []ports.Port{
-		{
+	ports := []PortWithTrunkDetails{{
+		Port: neutronports.Port{
 			Status: "ACTIVE",
-			FixedIPs: []ports.IP{
+			FixedIPs: []neutronports.IP{
 				{
 					IPAddress: "10.0.0.32",
 				},
@@ -541,9 +543,10 @@ func TestNodeAddressesCustomPublicNetworkWithIntersectingFixedIP(t *testing.T) {
 				},
 			},
 		},
+	},
 	}
 
-	addrs, err := nodeAddresses(&srv, ports, networkingOpts)
+	addrs, err := nodeAddresses(&srv, ports, nil, networkingOpts)
 	if err != nil {
 		t.Fatalf("nodeAddresses returned error: %v", err)
 	}
@@ -613,10 +616,10 @@ func TestNodeAddressesMultipleCustomInternalNetworks(t *testing.T) {
 		InternalNetworkName: []string{"private", "also-private"},
 	}
 
-	ports := []ports.Port{
-		{
+	ports := []PortWithTrunkDetails{{
+		Port: neutronports.Port{
 			Status: "ACTIVE",
-			FixedIPs: []ports.IP{
+			FixedIPs: []neutronports.IP{
 				{
 					IPAddress: "10.0.0.32",
 				},
@@ -625,9 +628,10 @@ func TestNodeAddressesMultipleCustomInternalNetworks(t *testing.T) {
 				},
 			},
 		},
+	},
 	}
 
-	addrs, err := nodeAddresses(&srv, ports, networkingOpts)
+	addrs, err := nodeAddresses(&srv, ports, nil, networkingOpts)
 	if err != nil {
 		t.Fatalf("nodeAddresses returned error: %v", err)
 	}
@@ -697,10 +701,10 @@ func TestNodeAddressesOneInternalNetwork(t *testing.T) {
 		InternalNetworkName: []string{"also-private"},
 	}
 
-	ports := []ports.Port{
-		{
+	ports := []PortWithTrunkDetails{{
+		Port: neutronports.Port{
 			Status: "ACTIVE",
-			FixedIPs: []ports.IP{
+			FixedIPs: []neutronports.IP{
 				{
 					IPAddress: "10.0.0.32",
 				},
@@ -709,9 +713,10 @@ func TestNodeAddressesOneInternalNetwork(t *testing.T) {
 				},
 			},
 		},
+	},
 	}
 
-	addrs, err := nodeAddresses(&srv, ports, networkingOpts)
+	addrs, err := nodeAddresses(&srv, ports, nil, networkingOpts)
 	if err != nil {
 		t.Fatalf("nodeAddresses returned error: %v", err)
 	}
@@ -773,10 +778,10 @@ func TestNodeAddressesIPv6Disabled(t *testing.T) {
 		IPv6SupportDisabled: true,
 	}
 
-	ports := []ports.Port{
-		{
+	ports := []PortWithTrunkDetails{{
+		Port: neutronports.Port{
 			Status: "ACTIVE",
-			FixedIPs: []ports.IP{
+			FixedIPs: []neutronports.IP{
 				{
 					IPAddress: "10.0.0.32",
 				},
@@ -785,9 +790,10 @@ func TestNodeAddressesIPv6Disabled(t *testing.T) {
 				},
 			},
 		},
+	},
 	}
 
-	addrs, err := nodeAddresses(&srv, ports, networkingOpts)
+	addrs, err := nodeAddresses(&srv, ports, nil, networkingOpts)
 	if err != nil {
 		t.Fatalf("nodeAddresses returned error: %v", err)
 	}
@@ -854,10 +860,10 @@ func TestNodeAddressesWithAddressSortOrderOptions(t *testing.T) {
 		AddressSortOrder:  "10.0.0.0/8, 50.56.176.0/24, 2001:4800::/32",
 	}
 
-	ports := []ports.Port{
-		{
+	ports := []PortWithTrunkDetails{{
+		Port: neutronports.Port{
 			Status: "ACTIVE",
-			FixedIPs: []ports.IP{
+			FixedIPs: []neutronports.IP{
 				{
 					IPAddress: "10.0.0.32",
 				},
@@ -866,9 +872,10 @@ func TestNodeAddressesWithAddressSortOrderOptions(t *testing.T) {
 				},
 			},
 		},
+	},
 	}
 
-	addrs, err := nodeAddresses(&srv, ports, networkingOpts)
+	addrs, err := nodeAddresses(&srv, ports, nil, networkingOpts)
 	if err != nil {
 		t.Fatalf("nodeAddresses returned error: %v", err)
 	}